Load necessary modules

In [1]:
# show images inline
%matplotlib inline

# automatically reload modules when they have changed
%load_ext autoreload
%autoreload 2

# import keras
import keras

# import keras_retinanet
import keras_retinanet
from keras_retinanet import models
from keras_retinanet.utils.image import read_image_bgr, preprocess_image, resize_image
from keras_retinanet.utils.visualization import draw_box, draw_caption
from keras_retinanet.utils.colors import label_color
from keras_retinanet.utils.gpu import setup_gpu

# import miscellaneous modules
import matplotlib.pyplot as plt
import cv2
import os
import time
import csv
import numpy as np
import xml.etree.ElementTree as ET
import pandas as pd
Using TensorFlow backend.
In [2]:
DATASET_DIR = './images'
ANNOTATIONS_FILE = 'annotations.csv'
CLASSES_FILE = 'classes.csv'

annotations = []
classes = set([])

for xml_file in [f for f in os.listdir(DATASET_DIR) if f.endswith(".xml")]:
  tree = ET.parse(os.path.join(DATASET_DIR, xml_file))
  root = tree.getroot()

  file_name = None

  for elem in root:
    if elem.tag == 'filename':
      file_name = os.path.join(DATASET_DIR, elem.text)

    if elem.tag == 'object':
      obj_name = None
      coords = []
      for subelem in elem:
        if subelem.tag == 'name':
          obj_name = subelem.text
        if subelem.tag == 'bndbox':
          for subsubelem in subelem:
            coords.append(subsubelem.text)
      item = [file_name] + coords + [obj_name]
      annotations.append(item)
      classes.add(obj_name)

with open(ANNOTATIONS_FILE, 'w') as f:
  writer = csv.writer(f)
  writer.writerows(annotations)

print('Successfully converted xml to annotations.')

with open(CLASSES_FILE, 'w') as f:
  for i, line in enumerate(classes):
    f.write('{}, {}\n'.format(line,i))
    
print('Successfully converted xml to classes.')
Successfully converted xml to annotations.
Successfully converted xml to classes.
In [3]:
annotations = pd.read_csv("annotations.csv")
annotations.head()
Out[3]:
./images/2020-06-22_16-06-25_CU.jpg 83 53 153 128 WBC
0 ./images/2020-06-22_16-06-25_CU.jpg 348 94 420 155 WBC
1 ./images/2020-06-22_16-06-25_CU.jpg 111 206 176 270 WBC
2 ./images/2020-06-22_16-06-25_CU.jpg 218 537 289 601 WBC
3 ./images/2020-06-22_16-06-25_CU.jpg 172 617 231 697 WBC
4 ./images/2020-06-22_16-06-25_CU.jpg 744 634 808 718 WBC
In [4]:
classes = pd.read_csv("classes.csv")
classes.head()
Out[4]:
Platelets 0
0 RBC 1
1 WBC 2
In [5]:
model_path = os.path.join('.', 'snapshots', 'resnet50_coco_best_v2.1.0.h5')

# load retinanet model
model = models.load_model(model_path, backbone_name='resnet50')

# if the model is not converted to an inference model, use the line below
# see: https://github.com/fizyr/keras-retinanet#converting-a-training-model-to-inference-model
#model = models.convert_model(model)

print(model.summary())
WARNING:tensorflow:From /home/microsisdcn/.virtualenvs/cv/lib/python3.6/site-packages/tensorflow_core/python/ops/resource_variable_ops.py:1630: calling BaseResourceVariable.__init__ (from tensorflow.python.ops.resource_variable_ops) with constraint is deprecated and will be removed in a future version.
Instructions for updating:
If using Keras pass *_constraint arguments to layers.
tracking <tf.Variable 'Variable:0' shape=(9, 4) dtype=float32> anchors
tracking <tf.Variable 'Variable_1:0' shape=(9, 4) dtype=float32> anchors
tracking <tf.Variable 'Variable_2:0' shape=(9, 4) dtype=float32> anchors
tracking <tf.Variable 'Variable_3:0' shape=(9, 4) dtype=float32> anchors
tracking <tf.Variable 'Variable_4:0' shape=(9, 4) dtype=float32> anchors
WARNING:tensorflow:From /home/microsisdcn/.virtualenvs/cv/lib/python3.6/site-packages/keras/backend/tensorflow_backend.py:4070: The name tf.nn.max_pool is deprecated. Please use tf.nn.max_pool2d instead.

WARNING:tensorflow:From /home/microsisdcn/Keras_Jupyter_Project/keras-retinanet/examples/keras_retinanet/backend/tensorflow_backend.py:104: where (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.
Instructions for updating:
Use tf.where in 2.0, which has the same broadcast rule as np.where
Model: "retinanet-bbox"
__________________________________________________________________________________________________
Layer (type)                    Output Shape         Param #     Connected to                     
==================================================================================================
input_1 (InputLayer)            (None, None, None, 3 0                                            
__________________________________________________________________________________________________
padding_conv1 (ZeroPadding2D)   (None, None, None, 3 0           input_1[0][0]                    
__________________________________________________________________________________________________
conv1 (Conv2D)                  (None, None, None, 6 9408        padding_conv1[0][0]              
__________________________________________________________________________________________________
bn_conv1 (BatchNormalization)   (None, None, None, 6 256         conv1[0][0]                      
__________________________________________________________________________________________________
conv1_relu (Activation)         (None, None, None, 6 0           bn_conv1[0][0]                   
__________________________________________________________________________________________________
pool1 (MaxPooling2D)            (None, None, None, 6 0           conv1_relu[0][0]                 
__________________________________________________________________________________________________
res2a_branch2a (Conv2D)         (None, None, None, 6 4096        pool1[0][0]                      
__________________________________________________________________________________________________
bn2a_branch2a (BatchNormalizati (None, None, None, 6 256         res2a_branch2a[0][0]             
__________________________________________________________________________________________________
res2a_branch2a_relu (Activation (None, None, None, 6 0           bn2a_branch2a[0][0]              
__________________________________________________________________________________________________
padding2a_branch2b (ZeroPadding (None, None, None, 6 0           res2a_branch2a_relu[0][0]        
__________________________________________________________________________________________________
res2a_branch2b (Conv2D)         (None, None, None, 6 36864       padding2a_branch2b[0][0]         
__________________________________________________________________________________________________
bn2a_branch2b (BatchNormalizati (None, None, None, 6 256         res2a_branch2b[0][0]             
__________________________________________________________________________________________________
res2a_branch2b_relu (Activation (None, None, None, 6 0           bn2a_branch2b[0][0]              
__________________________________________________________________________________________________
res2a_branch2c (Conv2D)         (None, None, None, 2 16384       res2a_branch2b_relu[0][0]        
__________________________________________________________________________________________________
res2a_branch1 (Conv2D)          (None, None, None, 2 16384       pool1[0][0]                      
__________________________________________________________________________________________________
bn2a_branch2c (BatchNormalizati (None, None, None, 2 1024        res2a_branch2c[0][0]             
__________________________________________________________________________________________________
bn2a_branch1 (BatchNormalizatio (None, None, None, 2 1024        res2a_branch1[0][0]              
__________________________________________________________________________________________________
res2a (Add)                     (None, None, None, 2 0           bn2a_branch2c[0][0]              
                                                                 bn2a_branch1[0][0]               
__________________________________________________________________________________________________
res2a_relu (Activation)         (None, None, None, 2 0           res2a[0][0]                      
__________________________________________________________________________________________________
res2b_branch2a (Conv2D)         (None, None, None, 6 16384       res2a_relu[0][0]                 
__________________________________________________________________________________________________
bn2b_branch2a (BatchNormalizati (None, None, None, 6 256         res2b_branch2a[0][0]             
__________________________________________________________________________________________________
res2b_branch2a_relu (Activation (None, None, None, 6 0           bn2b_branch2a[0][0]              
__________________________________________________________________________________________________
padding2b_branch2b (ZeroPadding (None, None, None, 6 0           res2b_branch2a_relu[0][0]        
__________________________________________________________________________________________________
res2b_branch2b (Conv2D)         (None, None, None, 6 36864       padding2b_branch2b[0][0]         
__________________________________________________________________________________________________
bn2b_branch2b (BatchNormalizati (None, None, None, 6 256         res2b_branch2b[0][0]             
__________________________________________________________________________________________________
res2b_branch2b_relu (Activation (None, None, None, 6 0           bn2b_branch2b[0][0]              
__________________________________________________________________________________________________
res2b_branch2c (Conv2D)         (None, None, None, 2 16384       res2b_branch2b_relu[0][0]        
__________________________________________________________________________________________________
bn2b_branch2c (BatchNormalizati (None, None, None, 2 1024        res2b_branch2c[0][0]             
__________________________________________________________________________________________________
res2b (Add)                     (None, None, None, 2 0           bn2b_branch2c[0][0]              
                                                                 res2a_relu[0][0]                 
__________________________________________________________________________________________________
res2b_relu (Activation)         (None, None, None, 2 0           res2b[0][0]                      
__________________________________________________________________________________________________
res2c_branch2a (Conv2D)         (None, None, None, 6 16384       res2b_relu[0][0]                 
__________________________________________________________________________________________________
bn2c_branch2a (BatchNormalizati (None, None, None, 6 256         res2c_branch2a[0][0]             
__________________________________________________________________________________________________
res2c_branch2a_relu (Activation (None, None, None, 6 0           bn2c_branch2a[0][0]              
__________________________________________________________________________________________________
padding2c_branch2b (ZeroPadding (None, None, None, 6 0           res2c_branch2a_relu[0][0]        
__________________________________________________________________________________________________
res2c_branch2b (Conv2D)         (None, None, None, 6 36864       padding2c_branch2b[0][0]         
__________________________________________________________________________________________________
bn2c_branch2b (BatchNormalizati (None, None, None, 6 256         res2c_branch2b[0][0]             
__________________________________________________________________________________________________
res2c_branch2b_relu (Activation (None, None, None, 6 0           bn2c_branch2b[0][0]              
__________________________________________________________________________________________________
res2c_branch2c (Conv2D)         (None, None, None, 2 16384       res2c_branch2b_relu[0][0]        
__________________________________________________________________________________________________
bn2c_branch2c (BatchNormalizati (None, None, None, 2 1024        res2c_branch2c[0][0]             
__________________________________________________________________________________________________
res2c (Add)                     (None, None, None, 2 0           bn2c_branch2c[0][0]              
                                                                 res2b_relu[0][0]                 
__________________________________________________________________________________________________
res2c_relu (Activation)         (None, None, None, 2 0           res2c[0][0]                      
__________________________________________________________________________________________________
res3a_branch2a (Conv2D)         (None, None, None, 1 32768       res2c_relu[0][0]                 
__________________________________________________________________________________________________
bn3a_branch2a (BatchNormalizati (None, None, None, 1 512         res3a_branch2a[0][0]             
__________________________________________________________________________________________________
res3a_branch2a_relu (Activation (None, None, None, 1 0           bn3a_branch2a[0][0]              
__________________________________________________________________________________________________
padding3a_branch2b (ZeroPadding (None, None, None, 1 0           res3a_branch2a_relu[0][0]        
__________________________________________________________________________________________________
res3a_branch2b (Conv2D)         (None, None, None, 1 147456      padding3a_branch2b[0][0]         
__________________________________________________________________________________________________
bn3a_branch2b (BatchNormalizati (None, None, None, 1 512         res3a_branch2b[0][0]             
__________________________________________________________________________________________________
res3a_branch2b_relu (Activation (None, None, None, 1 0           bn3a_branch2b[0][0]              
__________________________________________________________________________________________________
res3a_branch2c (Conv2D)         (None, None, None, 5 65536       res3a_branch2b_relu[0][0]        
__________________________________________________________________________________________________
res3a_branch1 (Conv2D)          (None, None, None, 5 131072      res2c_relu[0][0]                 
__________________________________________________________________________________________________
bn3a_branch2c (BatchNormalizati (None, None, None, 5 2048        res3a_branch2c[0][0]             
__________________________________________________________________________________________________
bn3a_branch1 (BatchNormalizatio (None, None, None, 5 2048        res3a_branch1[0][0]              
__________________________________________________________________________________________________
res3a (Add)                     (None, None, None, 5 0           bn3a_branch2c[0][0]              
                                                                 bn3a_branch1[0][0]               
__________________________________________________________________________________________________
res3a_relu (Activation)         (None, None, None, 5 0           res3a[0][0]                      
__________________________________________________________________________________________________
res3b_branch2a (Conv2D)         (None, None, None, 1 65536       res3a_relu[0][0]                 
__________________________________________________________________________________________________
bn3b_branch2a (BatchNormalizati (None, None, None, 1 512         res3b_branch2a[0][0]             
__________________________________________________________________________________________________
res3b_branch2a_relu (Activation (None, None, None, 1 0           bn3b_branch2a[0][0]              
__________________________________________________________________________________________________
padding3b_branch2b (ZeroPadding (None, None, None, 1 0           res3b_branch2a_relu[0][0]        
__________________________________________________________________________________________________
res3b_branch2b (Conv2D)         (None, None, None, 1 147456      padding3b_branch2b[0][0]         
__________________________________________________________________________________________________
bn3b_branch2b (BatchNormalizati (None, None, None, 1 512         res3b_branch2b[0][0]             
__________________________________________________________________________________________________
res3b_branch2b_relu (Activation (None, None, None, 1 0           bn3b_branch2b[0][0]              
__________________________________________________________________________________________________
res3b_branch2c (Conv2D)         (None, None, None, 5 65536       res3b_branch2b_relu[0][0]        
__________________________________________________________________________________________________
bn3b_branch2c (BatchNormalizati (None, None, None, 5 2048        res3b_branch2c[0][0]             
__________________________________________________________________________________________________
res3b (Add)                     (None, None, None, 5 0           bn3b_branch2c[0][0]              
                                                                 res3a_relu[0][0]                 
__________________________________________________________________________________________________
res3b_relu (Activation)         (None, None, None, 5 0           res3b[0][0]                      
__________________________________________________________________________________________________
res3c_branch2a (Conv2D)         (None, None, None, 1 65536       res3b_relu[0][0]                 
__________________________________________________________________________________________________
bn3c_branch2a (BatchNormalizati (None, None, None, 1 512         res3c_branch2a[0][0]             
__________________________________________________________________________________________________
res3c_branch2a_relu (Activation (None, None, None, 1 0           bn3c_branch2a[0][0]              
__________________________________________________________________________________________________
padding3c_branch2b (ZeroPadding (None, None, None, 1 0           res3c_branch2a_relu[0][0]        
__________________________________________________________________________________________________
res3c_branch2b (Conv2D)         (None, None, None, 1 147456      padding3c_branch2b[0][0]         
__________________________________________________________________________________________________
bn3c_branch2b (BatchNormalizati (None, None, None, 1 512         res3c_branch2b[0][0]             
__________________________________________________________________________________________________
res3c_branch2b_relu (Activation (None, None, None, 1 0           bn3c_branch2b[0][0]              
__________________________________________________________________________________________________
res3c_branch2c (Conv2D)         (None, None, None, 5 65536       res3c_branch2b_relu[0][0]        
__________________________________________________________________________________________________
bn3c_branch2c (BatchNormalizati (None, None, None, 5 2048        res3c_branch2c[0][0]             
__________________________________________________________________________________________________
res3c (Add)                     (None, None, None, 5 0           bn3c_branch2c[0][0]              
                                                                 res3b_relu[0][0]                 
__________________________________________________________________________________________________
res3c_relu (Activation)         (None, None, None, 5 0           res3c[0][0]                      
__________________________________________________________________________________________________
res3d_branch2a (Conv2D)         (None, None, None, 1 65536       res3c_relu[0][0]                 
__________________________________________________________________________________________________
bn3d_branch2a (BatchNormalizati (None, None, None, 1 512         res3d_branch2a[0][0]             
__________________________________________________________________________________________________
res3d_branch2a_relu (Activation (None, None, None, 1 0           bn3d_branch2a[0][0]              
__________________________________________________________________________________________________
padding3d_branch2b (ZeroPadding (None, None, None, 1 0           res3d_branch2a_relu[0][0]        
__________________________________________________________________________________________________
res3d_branch2b (Conv2D)         (None, None, None, 1 147456      padding3d_branch2b[0][0]         
__________________________________________________________________________________________________
bn3d_branch2b (BatchNormalizati (None, None, None, 1 512         res3d_branch2b[0][0]             
__________________________________________________________________________________________________
res3d_branch2b_relu (Activation (None, None, None, 1 0           bn3d_branch2b[0][0]              
__________________________________________________________________________________________________
res3d_branch2c (Conv2D)         (None, None, None, 5 65536       res3d_branch2b_relu[0][0]        
__________________________________________________________________________________________________
bn3d_branch2c (BatchNormalizati (None, None, None, 5 2048        res3d_branch2c[0][0]             
__________________________________________________________________________________________________
res3d (Add)                     (None, None, None, 5 0           bn3d_branch2c[0][0]              
                                                                 res3c_relu[0][0]                 
__________________________________________________________________________________________________
res3d_relu (Activation)         (None, None, None, 5 0           res3d[0][0]                      
__________________________________________________________________________________________________
res4a_branch2a (Conv2D)         (None, None, None, 2 131072      res3d_relu[0][0]                 
__________________________________________________________________________________________________
bn4a_branch2a (BatchNormalizati (None, None, None, 2 1024        res4a_branch2a[0][0]             
__________________________________________________________________________________________________
res4a_branch2a_relu (Activation (None, None, None, 2 0           bn4a_branch2a[0][0]              
__________________________________________________________________________________________________
padding4a_branch2b (ZeroPadding (None, None, None, 2 0           res4a_branch2a_relu[0][0]        
__________________________________________________________________________________________________
res4a_branch2b (Conv2D)         (None, None, None, 2 589824      padding4a_branch2b[0][0]         
__________________________________________________________________________________________________
bn4a_branch2b (BatchNormalizati (None, None, None, 2 1024        res4a_branch2b[0][0]             
__________________________________________________________________________________________________
res4a_branch2b_relu (Activation (None, None, None, 2 0           bn4a_branch2b[0][0]              
__________________________________________________________________________________________________
res4a_branch2c (Conv2D)         (None, None, None, 1 262144      res4a_branch2b_relu[0][0]        
__________________________________________________________________________________________________
res4a_branch1 (Conv2D)          (None, None, None, 1 524288      res3d_relu[0][0]                 
__________________________________________________________________________________________________
bn4a_branch2c (BatchNormalizati (None, None, None, 1 4096        res4a_branch2c[0][0]             
__________________________________________________________________________________________________
bn4a_branch1 (BatchNormalizatio (None, None, None, 1 4096        res4a_branch1[0][0]              
__________________________________________________________________________________________________
res4a (Add)                     (None, None, None, 1 0           bn4a_branch2c[0][0]              
                                                                 bn4a_branch1[0][0]               
__________________________________________________________________________________________________
res4a_relu (Activation)         (None, None, None, 1 0           res4a[0][0]                      
__________________________________________________________________________________________________
res4b_branch2a (Conv2D)         (None, None, None, 2 262144      res4a_relu[0][0]                 
__________________________________________________________________________________________________
bn4b_branch2a (BatchNormalizati (None, None, None, 2 1024        res4b_branch2a[0][0]             
__________________________________________________________________________________________________
res4b_branch2a_relu (Activation (None, None, None, 2 0           bn4b_branch2a[0][0]              
__________________________________________________________________________________________________
padding4b_branch2b (ZeroPadding (None, None, None, 2 0           res4b_branch2a_relu[0][0]        
__________________________________________________________________________________________________
res4b_branch2b (Conv2D)         (None, None, None, 2 589824      padding4b_branch2b[0][0]         
__________________________________________________________________________________________________
bn4b_branch2b (BatchNormalizati (None, None, None, 2 1024        res4b_branch2b[0][0]             
__________________________________________________________________________________________________
res4b_branch2b_relu (Activation (None, None, None, 2 0           bn4b_branch2b[0][0]              
__________________________________________________________________________________________________
res4b_branch2c (Conv2D)         (None, None, None, 1 262144      res4b_branch2b_relu[0][0]        
__________________________________________________________________________________________________
bn4b_branch2c (BatchNormalizati (None, None, None, 1 4096        res4b_branch2c[0][0]             
__________________________________________________________________________________________________
res4b (Add)                     (None, None, None, 1 0           bn4b_branch2c[0][0]              
                                                                 res4a_relu[0][0]                 
__________________________________________________________________________________________________
res4b_relu (Activation)         (None, None, None, 1 0           res4b[0][0]                      
__________________________________________________________________________________________________
res4c_branch2a (Conv2D)         (None, None, None, 2 262144      res4b_relu[0][0]                 
__________________________________________________________________________________________________
bn4c_branch2a (BatchNormalizati (None, None, None, 2 1024        res4c_branch2a[0][0]             
__________________________________________________________________________________________________
res4c_branch2a_relu (Activation (None, None, None, 2 0           bn4c_branch2a[0][0]              
__________________________________________________________________________________________________
padding4c_branch2b (ZeroPadding (None, None, None, 2 0           res4c_branch2a_relu[0][0]        
__________________________________________________________________________________________________
res4c_branch2b (Conv2D)         (None, None, None, 2 589824      padding4c_branch2b[0][0]         
__________________________________________________________________________________________________
bn4c_branch2b (BatchNormalizati (None, None, None, 2 1024        res4c_branch2b[0][0]             
__________________________________________________________________________________________________
res4c_branch2b_relu (Activation (None, None, None, 2 0           bn4c_branch2b[0][0]              
__________________________________________________________________________________________________
res4c_branch2c (Conv2D)         (None, None, None, 1 262144      res4c_branch2b_relu[0][0]        
__________________________________________________________________________________________________
bn4c_branch2c (BatchNormalizati (None, None, None, 1 4096        res4c_branch2c[0][0]             
__________________________________________________________________________________________________
res4c (Add)                     (None, None, None, 1 0           bn4c_branch2c[0][0]              
                                                                 res4b_relu[0][0]                 
__________________________________________________________________________________________________
res4c_relu (Activation)         (None, None, None, 1 0           res4c[0][0]                      
__________________________________________________________________________________________________
res4d_branch2a (Conv2D)         (None, None, None, 2 262144      res4c_relu[0][0]                 
__________________________________________________________________________________________________
bn4d_branch2a (BatchNormalizati (None, None, None, 2 1024        res4d_branch2a[0][0]             
__________________________________________________________________________________________________
res4d_branch2a_relu (Activation (None, None, None, 2 0           bn4d_branch2a[0][0]              
__________________________________________________________________________________________________
padding4d_branch2b (ZeroPadding (None, None, None, 2 0           res4d_branch2a_relu[0][0]        
__________________________________________________________________________________________________
res4d_branch2b (Conv2D)         (None, None, None, 2 589824      padding4d_branch2b[0][0]         
__________________________________________________________________________________________________
bn4d_branch2b (BatchNormalizati (None, None, None, 2 1024        res4d_branch2b[0][0]             
__________________________________________________________________________________________________
res4d_branch2b_relu (Activation (None, None, None, 2 0           bn4d_branch2b[0][0]              
__________________________________________________________________________________________________
res4d_branch2c (Conv2D)         (None, None, None, 1 262144      res4d_branch2b_relu[0][0]        
__________________________________________________________________________________________________
bn4d_branch2c (BatchNormalizati (None, None, None, 1 4096        res4d_branch2c[0][0]             
__________________________________________________________________________________________________
res4d (Add)                     (None, None, None, 1 0           bn4d_branch2c[0][0]              
                                                                 res4c_relu[0][0]                 
__________________________________________________________________________________________________
res4d_relu (Activation)         (None, None, None, 1 0           res4d[0][0]                      
__________________________________________________________________________________________________
res4e_branch2a (Conv2D)         (None, None, None, 2 262144      res4d_relu[0][0]                 
__________________________________________________________________________________________________
bn4e_branch2a (BatchNormalizati (None, None, None, 2 1024        res4e_branch2a[0][0]             
__________________________________________________________________________________________________
res4e_branch2a_relu (Activation (None, None, None, 2 0           bn4e_branch2a[0][0]              
__________________________________________________________________________________________________
padding4e_branch2b (ZeroPadding (None, None, None, 2 0           res4e_branch2a_relu[0][0]        
__________________________________________________________________________________________________
res4e_branch2b (Conv2D)         (None, None, None, 2 589824      padding4e_branch2b[0][0]         
__________________________________________________________________________________________________
bn4e_branch2b (BatchNormalizati (None, None, None, 2 1024        res4e_branch2b[0][0]             
__________________________________________________________________________________________________
res4e_branch2b_relu (Activation (None, None, None, 2 0           bn4e_branch2b[0][0]              
__________________________________________________________________________________________________
res4e_branch2c (Conv2D)         (None, None, None, 1 262144      res4e_branch2b_relu[0][0]        
__________________________________________________________________________________________________
bn4e_branch2c (BatchNormalizati (None, None, None, 1 4096        res4e_branch2c[0][0]             
__________________________________________________________________________________________________
res4e (Add)                     (None, None, None, 1 0           bn4e_branch2c[0][0]              
                                                                 res4d_relu[0][0]                 
__________________________________________________________________________________________________
res4e_relu (Activation)         (None, None, None, 1 0           res4e[0][0]                      
__________________________________________________________________________________________________
res4f_branch2a (Conv2D)         (None, None, None, 2 262144      res4e_relu[0][0]                 
__________________________________________________________________________________________________
bn4f_branch2a (BatchNormalizati (None, None, None, 2 1024        res4f_branch2a[0][0]             
__________________________________________________________________________________________________
res4f_branch2a_relu (Activation (None, None, None, 2 0           bn4f_branch2a[0][0]              
__________________________________________________________________________________________________
padding4f_branch2b (ZeroPadding (None, None, None, 2 0           res4f_branch2a_relu[0][0]        
__________________________________________________________________________________________________
res4f_branch2b (Conv2D)         (None, None, None, 2 589824      padding4f_branch2b[0][0]         
__________________________________________________________________________________________________
bn4f_branch2b (BatchNormalizati (None, None, None, 2 1024        res4f_branch2b[0][0]             
__________________________________________________________________________________________________
res4f_branch2b_relu (Activation (None, None, None, 2 0           bn4f_branch2b[0][0]              
__________________________________________________________________________________________________
res4f_branch2c (Conv2D)         (None, None, None, 1 262144      res4f_branch2b_relu[0][0]        
__________________________________________________________________________________________________
bn4f_branch2c (BatchNormalizati (None, None, None, 1 4096        res4f_branch2c[0][0]             
__________________________________________________________________________________________________
res4f (Add)                     (None, None, None, 1 0           bn4f_branch2c[0][0]              
                                                                 res4e_relu[0][0]                 
__________________________________________________________________________________________________
res4f_relu (Activation)         (None, None, None, 1 0           res4f[0][0]                      
__________________________________________________________________________________________________
res5a_branch2a (Conv2D)         (None, None, None, 5 524288      res4f_relu[0][0]                 
__________________________________________________________________________________________________
bn5a_branch2a (BatchNormalizati (None, None, None, 5 2048        res5a_branch2a[0][0]             
__________________________________________________________________________________________________
res5a_branch2a_relu (Activation (None, None, None, 5 0           bn5a_branch2a[0][0]              
__________________________________________________________________________________________________
padding5a_branch2b (ZeroPadding (None, None, None, 5 0           res5a_branch2a_relu[0][0]        
__________________________________________________________________________________________________
res5a_branch2b (Conv2D)         (None, None, None, 5 2359296     padding5a_branch2b[0][0]         
__________________________________________________________________________________________________
bn5a_branch2b (BatchNormalizati (None, None, None, 5 2048        res5a_branch2b[0][0]             
__________________________________________________________________________________________________
res5a_branch2b_relu (Activation (None, None, None, 5 0           bn5a_branch2b[0][0]              
__________________________________________________________________________________________________
res5a_branch2c (Conv2D)         (None, None, None, 2 1048576     res5a_branch2b_relu[0][0]        
__________________________________________________________________________________________________
res5a_branch1 (Conv2D)          (None, None, None, 2 2097152     res4f_relu[0][0]                 
__________________________________________________________________________________________________
bn5a_branch2c (BatchNormalizati (None, None, None, 2 8192        res5a_branch2c[0][0]             
__________________________________________________________________________________________________
bn5a_branch1 (BatchNormalizatio (None, None, None, 2 8192        res5a_branch1[0][0]              
__________________________________________________________________________________________________
res5a (Add)                     (None, None, None, 2 0           bn5a_branch2c[0][0]              
                                                                 bn5a_branch1[0][0]               
__________________________________________________________________________________________________
res5a_relu (Activation)         (None, None, None, 2 0           res5a[0][0]                      
__________________________________________________________________________________________________
res5b_branch2a (Conv2D)         (None, None, None, 5 1048576     res5a_relu[0][0]                 
__________________________________________________________________________________________________
bn5b_branch2a (BatchNormalizati (None, None, None, 5 2048        res5b_branch2a[0][0]             
__________________________________________________________________________________________________
res5b_branch2a_relu (Activation (None, None, None, 5 0           bn5b_branch2a[0][0]              
__________________________________________________________________________________________________
padding5b_branch2b (ZeroPadding (None, None, None, 5 0           res5b_branch2a_relu[0][0]        
__________________________________________________________________________________________________
res5b_branch2b (Conv2D)         (None, None, None, 5 2359296     padding5b_branch2b[0][0]         
__________________________________________________________________________________________________
bn5b_branch2b (BatchNormalizati (None, None, None, 5 2048        res5b_branch2b[0][0]             
__________________________________________________________________________________________________
res5b_branch2b_relu (Activation (None, None, None, 5 0           bn5b_branch2b[0][0]              
__________________________________________________________________________________________________
res5b_branch2c (Conv2D)         (None, None, None, 2 1048576     res5b_branch2b_relu[0][0]        
__________________________________________________________________________________________________
bn5b_branch2c (BatchNormalizati (None, None, None, 2 8192        res5b_branch2c[0][0]             
__________________________________________________________________________________________________
res5b (Add)                     (None, None, None, 2 0           bn5b_branch2c[0][0]              
                                                                 res5a_relu[0][0]                 
__________________________________________________________________________________________________
res5b_relu (Activation)         (None, None, None, 2 0           res5b[0][0]                      
__________________________________________________________________________________________________
res5c_branch2a (Conv2D)         (None, None, None, 5 1048576     res5b_relu[0][0]                 
__________________________________________________________________________________________________
bn5c_branch2a (BatchNormalizati (None, None, None, 5 2048        res5c_branch2a[0][0]             
__________________________________________________________________________________________________
res5c_branch2a_relu (Activation (None, None, None, 5 0           bn5c_branch2a[0][0]              
__________________________________________________________________________________________________
padding5c_branch2b (ZeroPadding (None, None, None, 5 0           res5c_branch2a_relu[0][0]        
__________________________________________________________________________________________________
res5c_branch2b (Conv2D)         (None, None, None, 5 2359296     padding5c_branch2b[0][0]         
__________________________________________________________________________________________________
bn5c_branch2b (BatchNormalizati (None, None, None, 5 2048        res5c_branch2b[0][0]             
__________________________________________________________________________________________________
res5c_branch2b_relu (Activation (None, None, None, 5 0           bn5c_branch2b[0][0]              
__________________________________________________________________________________________________
res5c_branch2c (Conv2D)         (None, None, None, 2 1048576     res5c_branch2b_relu[0][0]        
__________________________________________________________________________________________________
bn5c_branch2c (BatchNormalizati (None, None, None, 2 8192        res5c_branch2c[0][0]             
__________________________________________________________________________________________________
res5c (Add)                     (None, None, None, 2 0           bn5c_branch2c[0][0]              
                                                                 res5b_relu[0][0]                 
__________________________________________________________________________________________________
res5c_relu (Activation)         (None, None, None, 2 0           res5c[0][0]                      
__________________________________________________________________________________________________
C5_reduced (Conv2D)             (None, None, None, 2 524544      res5c_relu[0][0]                 
__________________________________________________________________________________________________
P5_upsampled (UpsampleLike)     (None, None, None, 2 0           C5_reduced[0][0]                 
                                                                 res4f_relu[0][0]                 
__________________________________________________________________________________________________
C4_reduced (Conv2D)             (None, None, None, 2 262400      res4f_relu[0][0]                 
__________________________________________________________________________________________________
P4_merged (Add)                 (None, None, None, 2 0           P5_upsampled[0][0]               
                                                                 C4_reduced[0][0]                 
__________________________________________________________________________________________________
P4_upsampled (UpsampleLike)     (None, None, None, 2 0           P4_merged[0][0]                  
                                                                 res3d_relu[0][0]                 
__________________________________________________________________________________________________
C3_reduced (Conv2D)             (None, None, None, 2 131328      res3d_relu[0][0]                 
__________________________________________________________________________________________________
P6 (Conv2D)                     (None, None, None, 2 4718848     res5c_relu[0][0]                 
__________________________________________________________________________________________________
P3_merged (Add)                 (None, None, None, 2 0           P4_upsampled[0][0]               
                                                                 C3_reduced[0][0]                 
__________________________________________________________________________________________________
C6_relu (Activation)            (None, None, None, 2 0           P6[0][0]                         
__________________________________________________________________________________________________
P3 (Conv2D)                     (None, None, None, 2 590080      P3_merged[0][0]                  
__________________________________________________________________________________________________
P4 (Conv2D)                     (None, None, None, 2 590080      P4_merged[0][0]                  
__________________________________________________________________________________________________
P5 (Conv2D)                     (None, None, None, 2 590080      C5_reduced[0][0]                 
__________________________________________________________________________________________________
P7 (Conv2D)                     (None, None, None, 2 590080      C6_relu[0][0]                    
__________________________________________________________________________________________________
anchors_0 (Anchors)             (None, None, 4)      0           P3[0][0]                         
__________________________________________________________________________________________________
anchors_1 (Anchors)             (None, None, 4)      0           P4[0][0]                         
__________________________________________________________________________________________________
anchors_2 (Anchors)             (None, None, 4)      0           P5[0][0]                         
__________________________________________________________________________________________________
anchors_3 (Anchors)             (None, None, 4)      0           P6[0][0]                         
__________________________________________________________________________________________________
anchors_4 (Anchors)             (None, None, 4)      0           P7[0][0]                         
__________________________________________________________________________________________________
regression_submodel (Model)     (None, None, 4)      2443300     P3[0][0]                         
                                                                 P4[0][0]                         
                                                                 P5[0][0]                         
                                                                 P6[0][0]                         
                                                                 P7[0][0]                         
__________________________________________________________________________________________________
anchors (Concatenate)           (None, None, 4)      0           anchors_0[0][0]                  
                                                                 anchors_1[0][0]                  
                                                                 anchors_2[0][0]                  
                                                                 anchors_3[0][0]                  
                                                                 anchors_4[0][0]                  
__________________________________________________________________________________________________
regression (Concatenate)        (None, None, 4)      0           regression_submodel[1][0]        
                                                                 regression_submodel[2][0]        
                                                                 regression_submodel[3][0]        
                                                                 regression_submodel[4][0]        
                                                                 regression_submodel[5][0]        
__________________________________________________________________________________________________
boxes (RegressBoxes)            (None, None, 4)      0           anchors[0][0]                    
                                                                 regression[0][0]                 
__________________________________________________________________________________________________
classification_submodel (Model) (None, None, 80)     4019920     P3[0][0]                         
                                                                 P4[0][0]                         
                                                                 P5[0][0]                         
                                                                 P6[0][0]                         
                                                                 P7[0][0]                         
__________________________________________________________________________________________________
clipped_boxes (ClipBoxes)       (None, None, 4)      0           input_1[0][0]                    
                                                                 boxes[0][0]                      
__________________________________________________________________________________________________
classification (Concatenate)    (None, None, 80)     0           classification_submodel[1][0]    
                                                                 classification_submodel[2][0]    
                                                                 classification_submodel[3][0]    
                                                                 classification_submodel[4][0]    
                                                                 classification_submodel[5][0]    
__________________________________________________________________________________________________
filtered_detections (FilterDete [(None, 300, 4), (No 0           clipped_boxes[0][0]              
                                                                 classification[0][0]             
==================================================================================================
Total params: 38,021,812
Trainable params: 37,915,572
Non-trainable params: 106,240
__________________________________________________________________________________________________
None
/home/microsisdcn/.virtualenvs/cv/lib/python3.6/site-packages/keras/engine/saving.py:341: UserWarning: No training configuration found in save file: the model was *not* compiled. Compile it manually.
  warnings.warn('No training configuration found in save file: '
In [6]:
#!keras_retinanet/bin/train.py --weights {model_path} --batch-size 8 --steps 500 --epochs 10 csv annotations.csv classes.csv

#work #python keras_retinanet/bin/train.py --weights ./snapshots/resnet50_coco_best_v2.1.0.h5 --gpu 1 --step 500 --epochs 1 csv annotations.csv classes.csv 
#work #multi-gpu python keras_retinanet/bin/train.py --weights ./snapshots/resnet50_coco_best_v2.1.0.h5 --freeze-backbone --random-transform --multi-gpu 2 --multi-gpu-force --batch-size 32 --epochs 100 --steps 1000 --weighted-average --compute-val-loss csv annotations.csv classes.csv 

!keras_retinanet/bin/train.py --weights ./snapshots/resnet50_coco_best_v2.1.0.h5 --freeze-backbone --random-transform --multi-gpu 2 --multi-gpu-force --batch-size 8 --epochs 100 --steps 1000 --weighted-average --compute-val-loss csv annotations.csv classes.csv 


#https://github.com/fizyr/keras-retinanet/issues/575
#python ./keras_retinanet/bin/convert_model.py ./snapshots/resnet50_csv_01.h5 ./snapshots/resnet50_csv_01_covert.h5 


#https://medium.com/@ringlayer/cardboard-box-detection-using-retinanet-keras-5d4f331d9d15
Using TensorFlow backend.
Creating model, this may take a second...
WARNING:tensorflow:From /home/microsisdcn/.virtualenvs/cv/lib/python3.6/site-packages/tensorflow_core/python/ops/resource_variable_ops.py:1630: calling BaseResourceVariable.__init__ (from tensorflow.python.ops.resource_variable_ops) with constraint is deprecated and will be removed in a future version.
Instructions for updating:
If using Keras pass *_constraint arguments to layers.
WARNING:tensorflow:From /home/microsisdcn/.virtualenvs/cv/lib/python3.6/site-packages/keras/backend/tensorflow_backend.py:4070: The name tf.nn.max_pool is deprecated. Please use tf.nn.max_pool2d instead.

/home/microsisdcn/.virtualenvs/cv/lib/python3.6/site-packages/keras/engine/saving.py:1319: UserWarning: Skipping loading of weights for layer classification_submodel due to mismatch in shape ((3, 3, 256, 27) vs (720, 256, 3, 3)).
  weight_values[i].shape))
/home/microsisdcn/.virtualenvs/cv/lib/python3.6/site-packages/keras/engine/saving.py:1319: UserWarning: Skipping loading of weights for layer classification_submodel due to mismatch in shape ((27,) vs (720,)).
  weight_values[i].shape))
2019-12-13 18:14:51.003127: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcuda.so.1
2019-12-13 18:14:51.007930: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2019-12-13 18:14:51.008286: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1618] Found device 0 with properties: 
name: GeForce GTX 1080 Ti major: 6 minor: 1 memoryClockRate(GHz): 1.683
pciBusID: 0000:0b:00.0
2019-12-13 18:14:51.008357: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2019-12-13 18:14:51.008995: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1618] Found device 1 with properties: 
name: GeForce GTX 1080 Ti major: 6 minor: 1 memoryClockRate(GHz): 1.683
pciBusID: 0000:0c:00.0
2019-12-13 18:14:51.009155: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.0
2019-12-13 18:14:51.009960: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10.0
2019-12-13 18:14:51.010757: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10.0
2019-12-13 18:14:51.010945: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10.0
2019-12-13 18:14:51.012025: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10.0
2019-12-13 18:14:51.012791: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10.0
2019-12-13 18:14:51.015061: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7
2019-12-13 18:14:51.015196: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2019-12-13 18:14:51.016456: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2019-12-13 18:14:51.017321: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2019-12-13 18:14:51.020533: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2019-12-13 18:14:51.021492: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1746] Adding visible gpu devices: 0, 1
2019-12-13 18:14:51.021862: I tensorflow/core/platform/cpu_feature_guard.cc:142] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX2 FMA
2019-12-13 18:14:51.052550: I tensorflow/core/platform/profile_utils/cpu_utils.cc:94] CPU Frequency: 3393240000 Hz
2019-12-13 18:14:51.053478: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x6fc9310 initialized for platform Host (this does not guarantee that XLA will be used). Devices:
2019-12-13 18:14:51.053509: I tensorflow/compiler/xla/service/service.cc:176]   StreamExecutor device (0): Host, Default Version
2019-12-13 18:14:51.205608: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2019-12-13 18:14:51.207084: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2019-12-13 18:14:51.207775: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x705bc60 initialized for platform CUDA (this does not guarantee that XLA will be used). Devices:
2019-12-13 18:14:51.207788: I tensorflow/compiler/xla/service/service.cc:176]   StreamExecutor device (0): GeForce GTX 1080 Ti, Compute Capability 6.1
2019-12-13 18:14:51.207796: I tensorflow/compiler/xla/service/service.cc:176]   StreamExecutor device (1): GeForce GTX 1080 Ti, Compute Capability 6.1
2019-12-13 18:14:51.213769: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2019-12-13 18:14:51.214123: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1618] Found device 0 with properties: 
name: GeForce GTX 1080 Ti major: 6 minor: 1 memoryClockRate(GHz): 1.683
pciBusID: 0000:0b:00.0
2019-12-13 18:14:51.214203: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2019-12-13 18:14:51.214929: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1618] Found device 1 with properties: 
name: GeForce GTX 1080 Ti major: 6 minor: 1 memoryClockRate(GHz): 1.683
pciBusID: 0000:0c:00.0
2019-12-13 18:14:51.214974: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.0
2019-12-13 18:14:51.214990: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10.0
2019-12-13 18:14:51.215004: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10.0
2019-12-13 18:14:51.215017: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10.0
2019-12-13 18:14:51.215030: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10.0
2019-12-13 18:14:51.215043: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10.0
2019-12-13 18:14:51.215056: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7
2019-12-13 18:14:51.215124: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2019-12-13 18:14:51.219166: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2019-12-13 18:14:51.219952: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2019-12-13 18:14:51.220340: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2019-12-13 18:14:51.221094: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1746] Adding visible gpu devices: 0, 1
2019-12-13 18:14:51.221135: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.0
2019-12-13 18:14:51.224168: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1159] Device interconnect StreamExecutor with strength 1 edge matrix:
2019-12-13 18:14:51.224204: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1165]      0 1 
2019-12-13 18:14:51.224220: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1178] 0:   N Y 
2019-12-13 18:14:51.224234: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1178] 1:   Y N 
2019-12-13 18:14:51.224568: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2019-12-13 18:14:51.225319: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2019-12-13 18:14:51.226118: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2019-12-13 18:14:51.226481: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1304] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 74 MB memory) -> physical GPU (device: 0, name: GeForce GTX 1080 Ti, pci bus id: 0000:0b:00.0, compute capability: 6.1)
2019-12-13 18:14:51.227062: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:983] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2019-12-13 18:14:51.231728: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1304] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:1 with 10349 MB memory) -> physical GPU (device: 1, name: GeForce GTX 1080 Ti, pci bus id: 0000:0c:00.0, compute capability: 6.1)
tracking <tf.Variable 'Variable:0' shape=(9, 4) dtype=float32> anchors
tracking <tf.Variable 'Variable_1:0' shape=(9, 4) dtype=float32> anchors
tracking <tf.Variable 'Variable_2:0' shape=(9, 4) dtype=float32> anchors
tracking <tf.Variable 'Variable_3:0' shape=(9, 4) dtype=float32> anchors
tracking <tf.Variable 'Variable_4:0' shape=(9, 4) dtype=float32> anchors
WARNING:tensorflow:From keras_retinanet/bin/../../keras_retinanet/backend/tensorflow_backend.py:104: where (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.
Instructions for updating:
Use tf.where in 2.0, which has the same broadcast rule as np.where
Model: "retinanet"
__________________________________________________________________________________________________
Layer (type)                    Output Shape         Param #     Connected to                     
==================================================================================================
input_1 (InputLayer)            (None, None, None, 3 0                                            
__________________________________________________________________________________________________
conv1 (Conv2D)                  (None, None, None, 6 9408        input_1[0][0]                    
__________________________________________________________________________________________________
bn_conv1 (BatchNormalization)   (None, None, None, 6 256         conv1[0][0]                      
__________________________________________________________________________________________________
conv1_relu (Activation)         (None, None, None, 6 0           bn_conv1[0][0]                   
__________________________________________________________________________________________________
pool1 (MaxPooling2D)            (None, None, None, 6 0           conv1_relu[0][0]                 
__________________________________________________________________________________________________
res2a_branch2a (Conv2D)         (None, None, None, 6 4096        pool1[0][0]                      
__________________________________________________________________________________________________
bn2a_branch2a (BatchNormalizati (None, None, None, 6 256         res2a_branch2a[0][0]             
__________________________________________________________________________________________________
res2a_branch2a_relu (Activation (None, None, None, 6 0           bn2a_branch2a[0][0]              
__________________________________________________________________________________________________
padding2a_branch2b (ZeroPadding (None, None, None, 6 0           res2a_branch2a_relu[0][0]        
__________________________________________________________________________________________________
res2a_branch2b (Conv2D)         (None, None, None, 6 36864       padding2a_branch2b[0][0]         
__________________________________________________________________________________________________
bn2a_branch2b (BatchNormalizati (None, None, None, 6 256         res2a_branch2b[0][0]             
__________________________________________________________________________________________________
res2a_branch2b_relu (Activation (None, None, None, 6 0           bn2a_branch2b[0][0]              
__________________________________________________________________________________________________
res2a_branch2c (Conv2D)         (None, None, None, 2 16384       res2a_branch2b_relu[0][0]        
__________________________________________________________________________________________________
res2a_branch1 (Conv2D)          (None, None, None, 2 16384       pool1[0][0]                      
__________________________________________________________________________________________________
bn2a_branch2c (BatchNormalizati (None, None, None, 2 1024        res2a_branch2c[0][0]             
__________________________________________________________________________________________________
bn2a_branch1 (BatchNormalizatio (None, None, None, 2 1024        res2a_branch1[0][0]              
__________________________________________________________________________________________________
res2a (Add)                     (None, None, None, 2 0           bn2a_branch2c[0][0]              
                                                                 bn2a_branch1[0][0]               
__________________________________________________________________________________________________
res2a_relu (Activation)         (None, None, None, 2 0           res2a[0][0]                      
__________________________________________________________________________________________________
res2b_branch2a (Conv2D)         (None, None, None, 6 16384       res2a_relu[0][0]                 
__________________________________________________________________________________________________
bn2b_branch2a (BatchNormalizati (None, None, None, 6 256         res2b_branch2a[0][0]             
__________________________________________________________________________________________________
res2b_branch2a_relu (Activation (None, None, None, 6 0           bn2b_branch2a[0][0]              
__________________________________________________________________________________________________
padding2b_branch2b (ZeroPadding (None, None, None, 6 0           res2b_branch2a_relu[0][0]        
__________________________________________________________________________________________________
res2b_branch2b (Conv2D)         (None, None, None, 6 36864       padding2b_branch2b[0][0]         
__________________________________________________________________________________________________
bn2b_branch2b (BatchNormalizati (None, None, None, 6 256         res2b_branch2b[0][0]             
__________________________________________________________________________________________________
res2b_branch2b_relu (Activation (None, None, None, 6 0           bn2b_branch2b[0][0]              
__________________________________________________________________________________________________
res2b_branch2c (Conv2D)         (None, None, None, 2 16384       res2b_branch2b_relu[0][0]        
__________________________________________________________________________________________________
bn2b_branch2c (BatchNormalizati (None, None, None, 2 1024        res2b_branch2c[0][0]             
__________________________________________________________________________________________________
res2b (Add)                     (None, None, None, 2 0           bn2b_branch2c[0][0]              
                                                                 res2a_relu[0][0]                 
__________________________________________________________________________________________________
res2b_relu (Activation)         (None, None, None, 2 0           res2b[0][0]                      
__________________________________________________________________________________________________
res2c_branch2a (Conv2D)         (None, None, None, 6 16384       res2b_relu[0][0]                 
__________________________________________________________________________________________________
bn2c_branch2a (BatchNormalizati (None, None, None, 6 256         res2c_branch2a[0][0]             
__________________________________________________________________________________________________
res2c_branch2a_relu (Activation (None, None, None, 6 0           bn2c_branch2a[0][0]              
__________________________________________________________________________________________________
padding2c_branch2b (ZeroPadding (None, None, None, 6 0           res2c_branch2a_relu[0][0]        
__________________________________________________________________________________________________
res2c_branch2b (Conv2D)         (None, None, None, 6 36864       padding2c_branch2b[0][0]         
__________________________________________________________________________________________________
bn2c_branch2b (BatchNormalizati (None, None, None, 6 256         res2c_branch2b[0][0]             
__________________________________________________________________________________________________
res2c_branch2b_relu (Activation (None, None, None, 6 0           bn2c_branch2b[0][0]              
__________________________________________________________________________________________________
res2c_branch2c (Conv2D)         (None, None, None, 2 16384       res2c_branch2b_relu[0][0]        
__________________________________________________________________________________________________
bn2c_branch2c (BatchNormalizati (None, None, None, 2 1024        res2c_branch2c[0][0]             
__________________________________________________________________________________________________
res2c (Add)                     (None, None, None, 2 0           bn2c_branch2c[0][0]              
                                                                 res2b_relu[0][0]                 
__________________________________________________________________________________________________
res2c_relu (Activation)         (None, None, None, 2 0           res2c[0][0]                      
__________________________________________________________________________________________________
res3a_branch2a (Conv2D)         (None, None, None, 1 32768       res2c_relu[0][0]                 
__________________________________________________________________________________________________
bn3a_branch2a (BatchNormalizati (None, None, None, 1 512         res3a_branch2a[0][0]             
__________________________________________________________________________________________________
res3a_branch2a_relu (Activation (None, None, None, 1 0           bn3a_branch2a[0][0]              
__________________________________________________________________________________________________
padding3a_branch2b (ZeroPadding (None, None, None, 1 0           res3a_branch2a_relu[0][0]        
__________________________________________________________________________________________________
res3a_branch2b (Conv2D)         (None, None, None, 1 147456      padding3a_branch2b[0][0]         
__________________________________________________________________________________________________
bn3a_branch2b (BatchNormalizati (None, None, None, 1 512         res3a_branch2b[0][0]             
__________________________________________________________________________________________________
res3a_branch2b_relu (Activation (None, None, None, 1 0           bn3a_branch2b[0][0]              
__________________________________________________________________________________________________
res3a_branch2c (Conv2D)         (None, None, None, 5 65536       res3a_branch2b_relu[0][0]        
__________________________________________________________________________________________________
res3a_branch1 (Conv2D)          (None, None, None, 5 131072      res2c_relu[0][0]                 
__________________________________________________________________________________________________
bn3a_branch2c (BatchNormalizati (None, None, None, 5 2048        res3a_branch2c[0][0]             
__________________________________________________________________________________________________
bn3a_branch1 (BatchNormalizatio (None, None, None, 5 2048        res3a_branch1[0][0]              
__________________________________________________________________________________________________
res3a (Add)                     (None, None, None, 5 0           bn3a_branch2c[0][0]              
                                                                 bn3a_branch1[0][0]               
__________________________________________________________________________________________________
res3a_relu (Activation)         (None, None, None, 5 0           res3a[0][0]                      
__________________________________________________________________________________________________
res3b_branch2a (Conv2D)         (None, None, None, 1 65536       res3a_relu[0][0]                 
__________________________________________________________________________________________________
bn3b_branch2a (BatchNormalizati (None, None, None, 1 512         res3b_branch2a[0][0]             
__________________________________________________________________________________________________
res3b_branch2a_relu (Activation (None, None, None, 1 0           bn3b_branch2a[0][0]              
__________________________________________________________________________________________________
padding3b_branch2b (ZeroPadding (None, None, None, 1 0           res3b_branch2a_relu[0][0]        
__________________________________________________________________________________________________
res3b_branch2b (Conv2D)         (None, None, None, 1 147456      padding3b_branch2b[0][0]         
__________________________________________________________________________________________________
bn3b_branch2b (BatchNormalizati (None, None, None, 1 512         res3b_branch2b[0][0]             
__________________________________________________________________________________________________
res3b_branch2b_relu (Activation (None, None, None, 1 0           bn3b_branch2b[0][0]              
__________________________________________________________________________________________________
res3b_branch2c (Conv2D)         (None, None, None, 5 65536       res3b_branch2b_relu[0][0]        
__________________________________________________________________________________________________
bn3b_branch2c (BatchNormalizati (None, None, None, 5 2048        res3b_branch2c[0][0]             
__________________________________________________________________________________________________
res3b (Add)                     (None, None, None, 5 0           bn3b_branch2c[0][0]              
                                                                 res3a_relu[0][0]                 
__________________________________________________________________________________________________
res3b_relu (Activation)         (None, None, None, 5 0           res3b[0][0]                      
__________________________________________________________________________________________________
res3c_branch2a (Conv2D)         (None, None, None, 1 65536       res3b_relu[0][0]                 
__________________________________________________________________________________________________
bn3c_branch2a (BatchNormalizati (None, None, None, 1 512         res3c_branch2a[0][0]             
__________________________________________________________________________________________________
res3c_branch2a_relu (Activation (None, None, None, 1 0           bn3c_branch2a[0][0]              
__________________________________________________________________________________________________
padding3c_branch2b (ZeroPadding (None, None, None, 1 0           res3c_branch2a_relu[0][0]        
__________________________________________________________________________________________________
res3c_branch2b (Conv2D)         (None, None, None, 1 147456      padding3c_branch2b[0][0]         
__________________________________________________________________________________________________
bn3c_branch2b (BatchNormalizati (None, None, None, 1 512         res3c_branch2b[0][0]             
__________________________________________________________________________________________________
res3c_branch2b_relu (Activation (None, None, None, 1 0           bn3c_branch2b[0][0]              
__________________________________________________________________________________________________
res3c_branch2c (Conv2D)         (None, None, None, 5 65536       res3c_branch2b_relu[0][0]        
__________________________________________________________________________________________________
bn3c_branch2c (BatchNormalizati (None, None, None, 5 2048        res3c_branch2c[0][0]             
__________________________________________________________________________________________________
res3c (Add)                     (None, None, None, 5 0           bn3c_branch2c[0][0]              
                                                                 res3b_relu[0][0]                 
__________________________________________________________________________________________________
res3c_relu (Activation)         (None, None, None, 5 0           res3c[0][0]                      
__________________________________________________________________________________________________
res3d_branch2a (Conv2D)         (None, None, None, 1 65536       res3c_relu[0][0]                 
__________________________________________________________________________________________________
bn3d_branch2a (BatchNormalizati (None, None, None, 1 512         res3d_branch2a[0][0]             
__________________________________________________________________________________________________
res3d_branch2a_relu (Activation (None, None, None, 1 0           bn3d_branch2a[0][0]              
__________________________________________________________________________________________________
padding3d_branch2b (ZeroPadding (None, None, None, 1 0           res3d_branch2a_relu[0][0]        
__________________________________________________________________________________________________
res3d_branch2b (Conv2D)         (None, None, None, 1 147456      padding3d_branch2b[0][0]         
__________________________________________________________________________________________________
bn3d_branch2b (BatchNormalizati (None, None, None, 1 512         res3d_branch2b[0][0]             
__________________________________________________________________________________________________
res3d_branch2b_relu (Activation (None, None, None, 1 0           bn3d_branch2b[0][0]              
__________________________________________________________________________________________________
res3d_branch2c (Conv2D)         (None, None, None, 5 65536       res3d_branch2b_relu[0][0]        
__________________________________________________________________________________________________
bn3d_branch2c (BatchNormalizati (None, None, None, 5 2048        res3d_branch2c[0][0]             
__________________________________________________________________________________________________
res3d (Add)                     (None, None, None, 5 0           bn3d_branch2c[0][0]              
                                                                 res3c_relu[0][0]                 
__________________________________________________________________________________________________
res3d_relu (Activation)         (None, None, None, 5 0           res3d[0][0]                      
__________________________________________________________________________________________________
res4a_branch2a (Conv2D)         (None, None, None, 2 131072      res3d_relu[0][0]                 
__________________________________________________________________________________________________
bn4a_branch2a (BatchNormalizati (None, None, None, 2 1024        res4a_branch2a[0][0]             
__________________________________________________________________________________________________
res4a_branch2a_relu (Activation (None, None, None, 2 0           bn4a_branch2a[0][0]              
__________________________________________________________________________________________________
padding4a_branch2b (ZeroPadding (None, None, None, 2 0           res4a_branch2a_relu[0][0]        
__________________________________________________________________________________________________
res4a_branch2b (Conv2D)         (None, None, None, 2 589824      padding4a_branch2b[0][0]         
__________________________________________________________________________________________________
bn4a_branch2b (BatchNormalizati (None, None, None, 2 1024        res4a_branch2b[0][0]             
__________________________________________________________________________________________________
res4a_branch2b_relu (Activation (None, None, None, 2 0           bn4a_branch2b[0][0]              
__________________________________________________________________________________________________
res4a_branch2c (Conv2D)         (None, None, None, 1 262144      res4a_branch2b_relu[0][0]        
__________________________________________________________________________________________________
res4a_branch1 (Conv2D)          (None, None, None, 1 524288      res3d_relu[0][0]                 
__________________________________________________________________________________________________
bn4a_branch2c (BatchNormalizati (None, None, None, 1 4096        res4a_branch2c[0][0]             
__________________________________________________________________________________________________
bn4a_branch1 (BatchNormalizatio (None, None, None, 1 4096        res4a_branch1[0][0]              
__________________________________________________________________________________________________
res4a (Add)                     (None, None, None, 1 0           bn4a_branch2c[0][0]              
                                                                 bn4a_branch1[0][0]               
__________________________________________________________________________________________________
res4a_relu (Activation)         (None, None, None, 1 0           res4a[0][0]                      
__________________________________________________________________________________________________
res4b_branch2a (Conv2D)         (None, None, None, 2 262144      res4a_relu[0][0]                 
__________________________________________________________________________________________________
bn4b_branch2a (BatchNormalizati (None, None, None, 2 1024        res4b_branch2a[0][0]             
__________________________________________________________________________________________________
res4b_branch2a_relu (Activation (None, None, None, 2 0           bn4b_branch2a[0][0]              
__________________________________________________________________________________________________
padding4b_branch2b (ZeroPadding (None, None, None, 2 0           res4b_branch2a_relu[0][0]        
__________________________________________________________________________________________________
res4b_branch2b (Conv2D)         (None, None, None, 2 589824      padding4b_branch2b[0][0]         
__________________________________________________________________________________________________
bn4b_branch2b (BatchNormalizati (None, None, None, 2 1024        res4b_branch2b[0][0]             
__________________________________________________________________________________________________
res4b_branch2b_relu (Activation (None, None, None, 2 0           bn4b_branch2b[0][0]              
__________________________________________________________________________________________________
res4b_branch2c (Conv2D)         (None, None, None, 1 262144      res4b_branch2b_relu[0][0]        
__________________________________________________________________________________________________
bn4b_branch2c (BatchNormalizati (None, None, None, 1 4096        res4b_branch2c[0][0]             
__________________________________________________________________________________________________
res4b (Add)                     (None, None, None, 1 0           bn4b_branch2c[0][0]              
                                                                 res4a_relu[0][0]                 
__________________________________________________________________________________________________
res4b_relu (Activation)         (None, None, None, 1 0           res4b[0][0]                      
__________________________________________________________________________________________________
res4c_branch2a (Conv2D)         (None, None, None, 2 262144      res4b_relu[0][0]                 
__________________________________________________________________________________________________
bn4c_branch2a (BatchNormalizati (None, None, None, 2 1024        res4c_branch2a[0][0]             
__________________________________________________________________________________________________
res4c_branch2a_relu (Activation (None, None, None, 2 0           bn4c_branch2a[0][0]              
__________________________________________________________________________________________________
padding4c_branch2b (ZeroPadding (None, None, None, 2 0           res4c_branch2a_relu[0][0]        
__________________________________________________________________________________________________
res4c_branch2b (Conv2D)         (None, None, None, 2 589824      padding4c_branch2b[0][0]         
__________________________________________________________________________________________________
bn4c_branch2b (BatchNormalizati (None, None, None, 2 1024        res4c_branch2b[0][0]             
__________________________________________________________________________________________________
res4c_branch2b_relu (Activation (None, None, None, 2 0           bn4c_branch2b[0][0]              
__________________________________________________________________________________________________
res4c_branch2c (Conv2D)         (None, None, None, 1 262144      res4c_branch2b_relu[0][0]        
__________________________________________________________________________________________________
bn4c_branch2c (BatchNormalizati (None, None, None, 1 4096        res4c_branch2c[0][0]             
__________________________________________________________________________________________________
res4c (Add)                     (None, None, None, 1 0           bn4c_branch2c[0][0]              
                                                                 res4b_relu[0][0]                 
__________________________________________________________________________________________________
res4c_relu (Activation)         (None, None, None, 1 0           res4c[0][0]                      
__________________________________________________________________________________________________
res4d_branch2a (Conv2D)         (None, None, None, 2 262144      res4c_relu[0][0]                 
__________________________________________________________________________________________________
bn4d_branch2a (BatchNormalizati (None, None, None, 2 1024        res4d_branch2a[0][0]             
__________________________________________________________________________________________________
res4d_branch2a_relu (Activation (None, None, None, 2 0           bn4d_branch2a[0][0]              
__________________________________________________________________________________________________
padding4d_branch2b (ZeroPadding (None, None, None, 2 0           res4d_branch2a_relu[0][0]        
__________________________________________________________________________________________________
res4d_branch2b (Conv2D)         (None, None, None, 2 589824      padding4d_branch2b[0][0]         
__________________________________________________________________________________________________
bn4d_branch2b (BatchNormalizati (None, None, None, 2 1024        res4d_branch2b[0][0]             
__________________________________________________________________________________________________
res4d_branch2b_relu (Activation (None, None, None, 2 0           bn4d_branch2b[0][0]              
__________________________________________________________________________________________________
res4d_branch2c (Conv2D)         (None, None, None, 1 262144      res4d_branch2b_relu[0][0]        
__________________________________________________________________________________________________
bn4d_branch2c (BatchNormalizati (None, None, None, 1 4096        res4d_branch2c[0][0]             
__________________________________________________________________________________________________
res4d (Add)                     (None, None, None, 1 0           bn4d_branch2c[0][0]              
                                                                 res4c_relu[0][0]                 
__________________________________________________________________________________________________
res4d_relu (Activation)         (None, None, None, 1 0           res4d[0][0]                      
__________________________________________________________________________________________________
res4e_branch2a (Conv2D)         (None, None, None, 2 262144      res4d_relu[0][0]                 
__________________________________________________________________________________________________
bn4e_branch2a (BatchNormalizati (None, None, None, 2 1024        res4e_branch2a[0][0]             
__________________________________________________________________________________________________
res4e_branch2a_relu (Activation (None, None, None, 2 0           bn4e_branch2a[0][0]              
__________________________________________________________________________________________________
padding4e_branch2b (ZeroPadding (None, None, None, 2 0           res4e_branch2a_relu[0][0]        
__________________________________________________________________________________________________
res4e_branch2b (Conv2D)         (None, None, None, 2 589824      padding4e_branch2b[0][0]         
__________________________________________________________________________________________________
bn4e_branch2b (BatchNormalizati (None, None, None, 2 1024        res4e_branch2b[0][0]             
__________________________________________________________________________________________________
res4e_branch2b_relu (Activation (None, None, None, 2 0           bn4e_branch2b[0][0]              
__________________________________________________________________________________________________
res4e_branch2c (Conv2D)         (None, None, None, 1 262144      res4e_branch2b_relu[0][0]        
__________________________________________________________________________________________________
bn4e_branch2c (BatchNormalizati (None, None, None, 1 4096        res4e_branch2c[0][0]             
__________________________________________________________________________________________________
res4e (Add)                     (None, None, None, 1 0           bn4e_branch2c[0][0]              
                                                                 res4d_relu[0][0]                 
__________________________________________________________________________________________________
res4e_relu (Activation)         (None, None, None, 1 0           res4e[0][0]                      
__________________________________________________________________________________________________
res4f_branch2a (Conv2D)         (None, None, None, 2 262144      res4e_relu[0][0]                 
__________________________________________________________________________________________________
bn4f_branch2a (BatchNormalizati (None, None, None, 2 1024        res4f_branch2a[0][0]             
__________________________________________________________________________________________________
res4f_branch2a_relu (Activation (None, None, None, 2 0           bn4f_branch2a[0][0]              
__________________________________________________________________________________________________
padding4f_branch2b (ZeroPadding (None, None, None, 2 0           res4f_branch2a_relu[0][0]        
__________________________________________________________________________________________________
res4f_branch2b (Conv2D)         (None, None, None, 2 589824      padding4f_branch2b[0][0]         
__________________________________________________________________________________________________
bn4f_branch2b (BatchNormalizati (None, None, None, 2 1024        res4f_branch2b[0][0]             
__________________________________________________________________________________________________
res4f_branch2b_relu (Activation (None, None, None, 2 0           bn4f_branch2b[0][0]              
__________________________________________________________________________________________________
res4f_branch2c (Conv2D)         (None, None, None, 1 262144      res4f_branch2b_relu[0][0]        
__________________________________________________________________________________________________
bn4f_branch2c (BatchNormalizati (None, None, None, 1 4096        res4f_branch2c[0][0]             
__________________________________________________________________________________________________
res4f (Add)                     (None, None, None, 1 0           bn4f_branch2c[0][0]              
                                                                 res4e_relu[0][0]                 
__________________________________________________________________________________________________
res4f_relu (Activation)         (None, None, None, 1 0           res4f[0][0]                      
__________________________________________________________________________________________________
res5a_branch2a (Conv2D)         (None, None, None, 5 524288      res4f_relu[0][0]                 
__________________________________________________________________________________________________
bn5a_branch2a (BatchNormalizati (None, None, None, 5 2048        res5a_branch2a[0][0]             
__________________________________________________________________________________________________
res5a_branch2a_relu (Activation (None, None, None, 5 0           bn5a_branch2a[0][0]              
__________________________________________________________________________________________________
padding5a_branch2b (ZeroPadding (None, None, None, 5 0           res5a_branch2a_relu[0][0]        
__________________________________________________________________________________________________
res5a_branch2b (Conv2D)         (None, None, None, 5 2359296     padding5a_branch2b[0][0]         
__________________________________________________________________________________________________
bn5a_branch2b (BatchNormalizati (None, None, None, 5 2048        res5a_branch2b[0][0]             
__________________________________________________________________________________________________
res5a_branch2b_relu (Activation (None, None, None, 5 0           bn5a_branch2b[0][0]              
__________________________________________________________________________________________________
res5a_branch2c (Conv2D)         (None, None, None, 2 1048576     res5a_branch2b_relu[0][0]        
__________________________________________________________________________________________________
res5a_branch1 (Conv2D)          (None, None, None, 2 2097152     res4f_relu[0][0]                 
__________________________________________________________________________________________________
bn5a_branch2c (BatchNormalizati (None, None, None, 2 8192        res5a_branch2c[0][0]             
__________________________________________________________________________________________________
bn5a_branch1 (BatchNormalizatio (None, None, None, 2 8192        res5a_branch1[0][0]              
__________________________________________________________________________________________________
res5a (Add)                     (None, None, None, 2 0           bn5a_branch2c[0][0]              
                                                                 bn5a_branch1[0][0]               
__________________________________________________________________________________________________
res5a_relu (Activation)         (None, None, None, 2 0           res5a[0][0]                      
__________________________________________________________________________________________________
res5b_branch2a (Conv2D)         (None, None, None, 5 1048576     res5a_relu[0][0]                 
__________________________________________________________________________________________________
bn5b_branch2a (BatchNormalizati (None, None, None, 5 2048        res5b_branch2a[0][0]             
__________________________________________________________________________________________________
res5b_branch2a_relu (Activation (None, None, None, 5 0           bn5b_branch2a[0][0]              
__________________________________________________________________________________________________
padding5b_branch2b (ZeroPadding (None, None, None, 5 0           res5b_branch2a_relu[0][0]        
__________________________________________________________________________________________________
res5b_branch2b (Conv2D)         (None, None, None, 5 2359296     padding5b_branch2b[0][0]         
__________________________________________________________________________________________________
bn5b_branch2b (BatchNormalizati (None, None, None, 5 2048        res5b_branch2b[0][0]             
__________________________________________________________________________________________________
res5b_branch2b_relu (Activation (None, None, None, 5 0           bn5b_branch2b[0][0]              
__________________________________________________________________________________________________
res5b_branch2c (Conv2D)         (None, None, None, 2 1048576     res5b_branch2b_relu[0][0]        
__________________________________________________________________________________________________
bn5b_branch2c (BatchNormalizati (None, None, None, 2 8192        res5b_branch2c[0][0]             
__________________________________________________________________________________________________
res5b (Add)                     (None, None, None, 2 0           bn5b_branch2c[0][0]              
                                                                 res5a_relu[0][0]                 
__________________________________________________________________________________________________
res5b_relu (Activation)         (None, None, None, 2 0           res5b[0][0]                      
__________________________________________________________________________________________________
res5c_branch2a (Conv2D)         (None, None, None, 5 1048576     res5b_relu[0][0]                 
__________________________________________________________________________________________________
bn5c_branch2a (BatchNormalizati (None, None, None, 5 2048        res5c_branch2a[0][0]             
__________________________________________________________________________________________________
res5c_branch2a_relu (Activation (None, None, None, 5 0           bn5c_branch2a[0][0]              
__________________________________________________________________________________________________
padding5c_branch2b (ZeroPadding (None, None, None, 5 0           res5c_branch2a_relu[0][0]        
__________________________________________________________________________________________________
res5c_branch2b (Conv2D)         (None, None, None, 5 2359296     padding5c_branch2b[0][0]         
__________________________________________________________________________________________________
bn5c_branch2b (BatchNormalizati (None, None, None, 5 2048        res5c_branch2b[0][0]             
__________________________________________________________________________________________________
res5c_branch2b_relu (Activation (None, None, None, 5 0           bn5c_branch2b[0][0]              
__________________________________________________________________________________________________
res5c_branch2c (Conv2D)         (None, None, None, 2 1048576     res5c_branch2b_relu[0][0]        
__________________________________________________________________________________________________
bn5c_branch2c (BatchNormalizati (None, None, None, 2 8192        res5c_branch2c[0][0]             
__________________________________________________________________________________________________
res5c (Add)                     (None, None, None, 2 0           bn5c_branch2c[0][0]              
                                                                 res5b_relu[0][0]                 
__________________________________________________________________________________________________
res5c_relu (Activation)         (None, None, None, 2 0           res5c[0][0]                      
__________________________________________________________________________________________________
C5_reduced (Conv2D)             (None, None, None, 2 524544      res5c_relu[0][0]                 
__________________________________________________________________________________________________
P5_upsampled (UpsampleLike)     (None, None, None, 2 0           C5_reduced[0][0]                 
                                                                 res4f_relu[0][0]                 
__________________________________________________________________________________________________
C4_reduced (Conv2D)             (None, None, None, 2 262400      res4f_relu[0][0]                 
__________________________________________________________________________________________________
P4_merged (Add)                 (None, None, None, 2 0           P5_upsampled[0][0]               
                                                                 C4_reduced[0][0]                 
__________________________________________________________________________________________________
P4_upsampled (UpsampleLike)     (None, None, None, 2 0           P4_merged[0][0]                  
                                                                 res3d_relu[0][0]                 
__________________________________________________________________________________________________
C3_reduced (Conv2D)             (None, None, None, 2 131328      res3d_relu[0][0]                 
__________________________________________________________________________________________________
P6 (Conv2D)                     (None, None, None, 2 4718848     res5c_relu[0][0]                 
__________________________________________________________________________________________________
P3_merged (Add)                 (None, None, None, 2 0           P4_upsampled[0][0]               
                                                                 C3_reduced[0][0]                 
__________________________________________________________________________________________________
C6_relu (Activation)            (None, None, None, 2 0           P6[0][0]                         
__________________________________________________________________________________________________
P3 (Conv2D)                     (None, None, None, 2 590080      P3_merged[0][0]                  
__________________________________________________________________________________________________
P4 (Conv2D)                     (None, None, None, 2 590080      P4_merged[0][0]                  
__________________________________________________________________________________________________
P5 (Conv2D)                     (None, None, None, 2 590080      C5_reduced[0][0]                 
__________________________________________________________________________________________________
P7 (Conv2D)                     (None, None, None, 2 590080      C6_relu[0][0]                    
__________________________________________________________________________________________________
regression_submodel (Model)     (None, None, 4)      2443300     P3[0][0]                         
                                                                 P4[0][0]                         
                                                                 P5[0][0]                         
                                                                 P6[0][0]                         
                                                                 P7[0][0]                         
__________________________________________________________________________________________________
classification_submodel (Model) (None, None, 3)      2422555     P3[0][0]                         
                                                                 P4[0][0]                         
                                                                 P5[0][0]                         
                                                                 P6[0][0]                         
                                                                 P7[0][0]                         
__________________________________________________________________________________________________
regression (Concatenate)        (None, None, 4)      0           regression_submodel[1][0]        
                                                                 regression_submodel[2][0]        
                                                                 regression_submodel[3][0]        
                                                                 regression_submodel[4][0]        
                                                                 regression_submodel[5][0]        
__________________________________________________________________________________________________
classification (Concatenate)    (None, None, 3)      0           classification_submodel[1][0]    
                                                                 classification_submodel[2][0]    
                                                                 classification_submodel[3][0]    
                                                                 classification_submodel[4][0]    
                                                                 classification_submodel[5][0]    
==================================================================================================
Total params: 36,424,447
Trainable params: 12,863,295
Non-trainable params: 23,561,152
__________________________________________________________________________________________________
None
WARNING:tensorflow:From /home/microsisdcn/.virtualenvs/cv/lib/python3.6/site-packages/keras/backend/tensorflow_backend.py:422: The name tf.global_variables is deprecated. Please use tf.compat.v1.global_variables instead.

WARNING:tensorflow:From /home/microsisdcn/.virtualenvs/cv/lib/python3.6/site-packages/keras/callbacks/tensorboard_v1.py:200: The name tf.summary.merge_all is deprecated. Please use tf.compat.v1.summary.merge_all instead.

WARNING:tensorflow:From /home/microsisdcn/.virtualenvs/cv/lib/python3.6/site-packages/keras/callbacks/tensorboard_v1.py:203: The name tf.summary.FileWriter is deprecated. Please use tf.compat.v1.summary.FileWriter instead.

Epoch 1/100
2019-12-13 18:15:03.429707: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7
2019-12-13 18:15:04.121713: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10.0
2019-12-13 18:15:13.033707: W tensorflow/core/common_runtime/bfc_allocator.cc:419] Allocator (GPU_0_bfc) ran out of memory trying to allocate 1.0KiB (rounded to 1024).  Current allocation summary follows.
2019-12-13 18:15:13.033773: I tensorflow/core/common_runtime/bfc_allocator.cc:869] Bin (256): 	Total Chunks: 32, Chunks in use: 32. 8.0KiB allocated for chunks. 8.0KiB in use in bin. 1.1KiB client-requested in use in bin.
2019-12-13 18:15:13.033794: I tensorflow/core/common_runtime/bfc_allocator.cc:869] Bin (512): 	Total Chunks: 0, Chunks in use: 0. 0B allocated for chunks. 0B in use in bin. 0B client-requested in use in bin.
2019-12-13 18:15:13.033810: I tensorflow/core/common_runtime/bfc_allocator.cc:869] Bin (1024): 	Total Chunks: 25, Chunks in use: 25. 25.2KiB allocated for chunks. 25.2KiB in use in bin. 25.0KiB client-requested in use in bin.
2019-12-13 18:15:13.033826: I tensorflow/core/common_runtime/bfc_allocator.cc:869] Bin (2048): 	Total Chunks: 14, Chunks in use: 14. 28.0KiB allocated for chunks. 28.0KiB in use in bin. 28.0KiB client-requested in use in bin.
2019-12-13 18:15:13.033862: I tensorflow/core/common_runtime/bfc_allocator.cc:869] Bin (4096): 	Total Chunks: 10, Chunks in use: 10. 40.0KiB allocated for chunks. 40.0KiB in use in bin. 40.0KiB client-requested in use in bin.
2019-12-13 18:15:13.033880: I tensorflow/core/common_runtime/bfc_allocator.cc:869] Bin (8192): 	Total Chunks: 9, Chunks in use: 9. 72.0KiB allocated for chunks. 72.0KiB in use in bin. 72.0KiB client-requested in use in bin.
2019-12-13 18:15:13.033895: I tensorflow/core/common_runtime/bfc_allocator.cc:869] Bin (16384): 	Total Chunks: 0, Chunks in use: 0. 0B allocated for chunks. 0B in use in bin. 0B client-requested in use in bin.
2019-12-13 18:15:13.033911: I tensorflow/core/common_runtime/bfc_allocator.cc:869] Bin (32768): 	Total Chunks: 0, Chunks in use: 0. 0B allocated for chunks. 0B in use in bin. 0B client-requested in use in bin.
2019-12-13 18:15:13.033925: I tensorflow/core/common_runtime/bfc_allocator.cc:869] Bin (65536): 	Total Chunks: 0, Chunks in use: 0. 0B allocated for chunks. 0B in use in bin. 0B client-requested in use in bin.
2019-12-13 18:15:13.033942: I tensorflow/core/common_runtime/bfc_allocator.cc:869] Bin (131072): 	Total Chunks: 1, Chunks in use: 1. 243.0KiB allocated for chunks. 243.0KiB in use in bin. 243.0KiB client-requested in use in bin.
2019-12-13 18:15:13.033960: I tensorflow/core/common_runtime/bfc_allocator.cc:869] Bin (262144): 	Total Chunks: 1, Chunks in use: 1. 324.0KiB allocated for chunks. 324.0KiB in use in bin. 324.0KiB client-requested in use in bin.
2019-12-13 18:15:13.033977: I tensorflow/core/common_runtime/bfc_allocator.cc:869] Bin (524288): 	Total Chunks: 1, Chunks in use: 1. 512.0KiB allocated for chunks. 512.0KiB in use in bin. 512.0KiB client-requested in use in bin.
2019-12-13 18:15:13.033993: I tensorflow/core/common_runtime/bfc_allocator.cc:869] Bin (1048576): 	Total Chunks: 8, Chunks in use: 8. 8.00MiB allocated for chunks. 8.00MiB in use in bin. 8.00MiB client-requested in use in bin.
2019-12-13 18:15:13.034011: I tensorflow/core/common_runtime/bfc_allocator.cc:869] Bin (2097152): 	Total Chunks: 5, Chunks in use: 5. 10.75MiB allocated for chunks. 10.75MiB in use in bin. 10.75MiB client-requested in use in bin.
2019-12-13 18:15:13.034027: I tensorflow/core/common_runtime/bfc_allocator.cc:869] Bin (4194304): 	Total Chunks: 3, Chunks in use: 3. 12.00MiB allocated for chunks. 12.00MiB in use in bin. 12.00MiB client-requested in use in bin.
2019-12-13 18:15:13.034044: I tensorflow/core/common_runtime/bfc_allocator.cc:869] Bin (8388608): 	Total Chunks: 1, Chunks in use: 1. 8.00MiB allocated for chunks. 8.00MiB in use in bin. 8.00MiB client-requested in use in bin.
2019-12-13 18:15:13.034061: I tensorflow/core/common_runtime/bfc_allocator.cc:869] Bin (16777216): 	Total Chunks: 2, Chunks in use: 2. 34.40MiB allocated for chunks. 34.40MiB in use in bin. 27.00MiB client-requested in use in bin.
2019-12-13 18:15:13.034075: I tensorflow/core/common_runtime/bfc_allocator.cc:869] Bin (33554432): 	Total Chunks: 0, Chunks in use: 0. 0B allocated for chunks. 0B in use in bin. 0B client-requested in use in bin.
2019-12-13 18:15:13.034089: I tensorflow/core/common_runtime/bfc_allocator.cc:869] Bin (67108864): 	Total Chunks: 0, Chunks in use: 0. 0B allocated for chunks. 0B in use in bin. 0B client-requested in use in bin.
2019-12-13 18:15:13.034103: I tensorflow/core/common_runtime/bfc_allocator.cc:869] Bin (134217728): 	Total Chunks: 0, Chunks in use: 0. 0B allocated for chunks. 0B in use in bin. 0B client-requested in use in bin.
2019-12-13 18:15:13.034117: I tensorflow/core/common_runtime/bfc_allocator.cc:869] Bin (268435456): 	Total Chunks: 0, Chunks in use: 0. 0B allocated for chunks. 0B in use in bin. 0B client-requested in use in bin.
2019-12-13 18:15:13.034132: I tensorflow/core/common_runtime/bfc_allocator.cc:885] Bin for 1.0KiB was 1.0KiB, Chunk State: 
2019-12-13 18:15:13.034146: I tensorflow/core/common_runtime/bfc_allocator.cc:898] Next region of size 77987840
2019-12-13 18:15:13.034160: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa416000000 next 1 of size 1280
2019-12-13 18:15:13.034173: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa416000500 next 2 of size 256
2019-12-13 18:15:13.034184: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa416000600 next 3 of size 256
2019-12-13 18:15:13.034195: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa416000700 next 4 of size 256
2019-12-13 18:15:13.034207: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa416000800 next 5 of size 1024
2019-12-13 18:15:13.034219: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa416000c00 next 6 of size 256
2019-12-13 18:15:13.034230: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa416000d00 next 7 of size 256
2019-12-13 18:15:13.034241: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa416000e00 next 8 of size 256
2019-12-13 18:15:13.034252: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa416000f00 next 9 of size 256
2019-12-13 18:15:13.034263: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa416001000 next 10 of size 256
2019-12-13 18:15:13.034274: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa416001100 next 11 of size 256
2019-12-13 18:15:13.034285: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa416001200 next 12 of size 256
2019-12-13 18:15:13.034295: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa416001300 next 13 of size 256
2019-12-13 18:15:13.034306: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa416001400 next 14 of size 256
2019-12-13 18:15:13.034318: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa416001500 next 15 of size 524288
2019-12-13 18:15:13.034330: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa416081500 next 16 of size 18874368
2019-12-13 18:15:13.034343: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa417281500 next 17 of size 256
2019-12-13 18:15:13.034355: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa417281600 next 18 of size 2359296
2019-12-13 18:15:13.034366: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa4174c1600 next 19 of size 331776
2019-12-13 18:15:13.034378: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa417512600 next 20 of size 248832
2019-12-13 18:15:13.034390: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa41754f200 next 21 of size 2097152
2019-12-13 18:15:13.034401: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa41774f200 next 22 of size 1048576
2019-12-13 18:15:13.034412: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa41784f200 next 23 of size 256
2019-12-13 18:15:13.034423: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa41784f300 next 24 of size 256
2019-12-13 18:15:13.034434: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa41784f400 next 25 of size 256
2019-12-13 18:15:13.034445: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa41784f500 next 26 of size 256
2019-12-13 18:15:13.034456: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa41784f600 next 27 of size 256
2019-12-13 18:15:13.034467: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa41784f700 next 28 of size 256
2019-12-13 18:15:13.034478: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa41784f800 next 29 of size 256
2019-12-13 18:15:13.034489: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa41784f900 next 30 of size 256
2019-12-13 18:15:13.034500: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa41784fa00 next 31 of size 256
2019-12-13 18:15:13.034510: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa41784fb00 next 32 of size 256
2019-12-13 18:15:13.034521: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa41784fc00 next 33 of size 256
2019-12-13 18:15:13.034532: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa41784fd00 next 34 of size 256
2019-12-13 18:15:13.034543: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa41784fe00 next 35 of size 256
2019-12-13 18:15:13.034553: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa41784ff00 next 36 of size 256
2019-12-13 18:15:13.034565: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa417850000 next 37 of size 256
2019-12-13 18:15:13.034576: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa417850100 next 38 of size 256
2019-12-13 18:15:13.034586: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa417850200 next 39 of size 256
2019-12-13 18:15:13.034597: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa417850300 next 40 of size 256
2019-12-13 18:15:13.034608: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa417850400 next 41 of size 256
2019-12-13 18:15:13.034620: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa417850500 next 42 of size 4096
2019-12-13 18:15:13.034631: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa417851500 next 43 of size 4096
2019-12-13 18:15:13.034642: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa417852500 next 44 of size 1024
2019-12-13 18:15:13.034653: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa417852900 next 45 of size 1024
2019-12-13 18:15:13.034664: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa417852d00 next 46 of size 1024
2019-12-13 18:15:13.034675: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa417853100 next 47 of size 4096
2019-12-13 18:15:13.034686: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa417854100 next 48 of size 1048576
2019-12-13 18:15:13.034697: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa417954100 next 49 of size 4096
2019-12-13 18:15:13.034708: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa417955100 next 50 of size 1024
2019-12-13 18:15:13.034719: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa417955500 next 51 of size 1024
2019-12-13 18:15:13.034730: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa417955900 next 52 of size 1024
2019-12-13 18:15:13.034741: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa417955d00 next 53 of size 4096
2019-12-13 18:15:13.034752: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa417956d00 next 54 of size 4096
2019-12-13 18:15:13.034763: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa417957d00 next 55 of size 1048576
2019-12-13 18:15:13.034774: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa417a57d00 next 56 of size 1024
2019-12-13 18:15:13.034786: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa417a58100 next 57 of size 8192
2019-12-13 18:15:13.034797: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa417a5a100 next 58 of size 1024
2019-12-13 18:15:13.034808: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa417a5a500 next 59 of size 1048576
2019-12-13 18:15:13.034820: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa417b5a500 next 60 of size 2048
2019-12-13 18:15:13.034832: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa417b5ad00 next 61 of size 4194304
2019-12-13 18:15:13.034843: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa417f5ad00 next 62 of size 2048
2019-12-13 18:15:13.034854: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa417f5b500 next 63 of size 2048
2019-12-13 18:15:13.034866: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa417f5bd00 next 64 of size 2048
2019-12-13 18:15:13.034877: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa417f5c500 next 65 of size 4194304
2019-12-13 18:15:13.034888: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa41835c500 next 66 of size 2048
2019-12-13 18:15:13.034899: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa41835cd00 next 67 of size 4194304
2019-12-13 18:15:13.034910: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa41875cd00 next 68 of size 2048
2019-12-13 18:15:13.034922: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa41875d500 next 69 of size 1024
2019-12-13 18:15:13.034933: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa41875d900 next 70 of size 1024
2019-12-13 18:15:13.034943: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa41875dd00 next 71 of size 1024
2019-12-13 18:15:13.034954: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa41875e100 next 72 of size 2048
2019-12-13 18:15:13.034965: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa41875e900 next 73 of size 1024
2019-12-13 18:15:13.034976: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa41875ed00 next 74 of size 2048
2019-12-13 18:15:13.034986: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa41875f500 next 75 of size 2359296
2019-12-13 18:15:13.034997: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa41899f500 next 76 of size 4096
2019-12-13 18:15:13.035008: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa4189a0500 next 77 of size 2359296
2019-12-13 18:15:13.035019: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa418be0500 next 78 of size 1024
2019-12-13 18:15:13.035030: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa418be0900 next 79 of size 1024
2019-12-13 18:15:13.035041: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa418be0d00 next 80 of size 1048576
2019-12-13 18:15:13.035052: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa418ce0d00 next 81 of size 8388608
2019-12-13 18:15:13.035063: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa4194e0d00 next 82 of size 2048
2019-12-13 18:15:13.035074: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa4194e1500 next 83 of size 8192
2019-12-13 18:15:13.035085: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa4194e3500 next 84 of size 2048
2019-12-13 18:15:13.035096: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa4194e3d00 next 85 of size 8192
2019-12-13 18:15:13.035107: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa4194e5d00 next 86 of size 2048
2019-12-13 18:15:13.035118: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa4194e6500 next 87 of size 8192
2019-12-13 18:15:13.035129: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa4194e8500 next 88 of size 1024
2019-12-13 18:15:13.035140: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa4194e8900 next 89 of size 4096
2019-12-13 18:15:13.035152: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa4194e9900 next 90 of size 1024
2019-12-13 18:15:13.035163: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa4194e9d00 next 91 of size 2048
2019-12-13 18:15:13.035174: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa4194ea500 next 92 of size 8192
2019-12-13 18:15:13.035186: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa4194ec500 next 93 of size 2048
2019-12-13 18:15:13.035197: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa4194ecd00 next 94 of size 8192
2019-12-13 18:15:13.035208: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa4194eed00 next 95 of size 1024
2019-12-13 18:15:13.035219: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa4194ef100 next 96 of size 1024
2019-12-13 18:15:13.035231: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa4194ef500 next 97 of size 4096
2019-12-13 18:15:13.035242: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa4194f0500 next 98 of size 1024
2019-12-13 18:15:13.035253: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa4194f0900 next 99 of size 1048576
2019-12-13 18:15:13.035264: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa4195f0900 next 100 of size 2048
2019-12-13 18:15:13.035275: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa4195f1100 next 101 of size 1048576
2019-12-13 18:15:13.035287: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa4196f1100 next 102 of size 1024
2019-12-13 18:15:13.035298: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa4196f1500 next 103 of size 8192
2019-12-13 18:15:13.035308: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa4196f3500 next 104 of size 1024
2019-12-13 18:15:13.035319: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa4196f3900 next 105 of size 1024
2019-12-13 18:15:13.035330: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa4196f3d00 next 106 of size 4096
2019-12-13 18:15:13.035341: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa4196f4d00 next 107 of size 2097152
2019-12-13 18:15:13.035353: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa4198f4d00 next 108 of size 1048576
2019-12-13 18:15:13.035364: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa4199f4d00 next 109 of size 8192
2019-12-13 18:15:13.035375: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa4199f6d00 next 110 of size 1024
2019-12-13 18:15:13.035386: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa4199f7100 next 111 of size 8192
2019-12-13 18:15:13.035398: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa4199f9100 next 18446744073709551615 of size 17198848
2019-12-13 18:15:13.035409: I tensorflow/core/common_runtime/bfc_allocator.cc:914]      Summary of in-use Chunks by size: 
2019-12-13 18:15:13.035422: I tensorflow/core/common_runtime/bfc_allocator.cc:917] 32 Chunks of size 256 totalling 8.0KiB
2019-12-13 18:15:13.035435: I tensorflow/core/common_runtime/bfc_allocator.cc:917] 24 Chunks of size 1024 totalling 24.0KiB
2019-12-13 18:15:13.035447: I tensorflow/core/common_runtime/bfc_allocator.cc:917] 1 Chunks of size 1280 totalling 1.2KiB
2019-12-13 18:15:13.035460: I tensorflow/core/common_runtime/bfc_allocator.cc:917] 14 Chunks of size 2048 totalling 28.0KiB
2019-12-13 18:15:13.035473: I tensorflow/core/common_runtime/bfc_allocator.cc:917] 10 Chunks of size 4096 totalling 40.0KiB
2019-12-13 18:15:13.035485: I tensorflow/core/common_runtime/bfc_allocator.cc:917] 9 Chunks of size 8192 totalling 72.0KiB
2019-12-13 18:15:13.035497: I tensorflow/core/common_runtime/bfc_allocator.cc:917] 1 Chunks of size 248832 totalling 243.0KiB
2019-12-13 18:15:13.035510: I tensorflow/core/common_runtime/bfc_allocator.cc:917] 1 Chunks of size 331776 totalling 324.0KiB
2019-12-13 18:15:13.035521: I tensorflow/core/common_runtime/bfc_allocator.cc:917] 1 Chunks of size 524288 totalling 512.0KiB
2019-12-13 18:15:13.035534: I tensorflow/core/common_runtime/bfc_allocator.cc:917] 8 Chunks of size 1048576 totalling 8.00MiB
2019-12-13 18:15:13.035546: I tensorflow/core/common_runtime/bfc_allocator.cc:917] 2 Chunks of size 2097152 totalling 4.00MiB
2019-12-13 18:15:13.035558: I tensorflow/core/common_runtime/bfc_allocator.cc:917] 3 Chunks of size 2359296 totalling 6.75MiB
2019-12-13 18:15:13.035571: I tensorflow/core/common_runtime/bfc_allocator.cc:917] 3 Chunks of size 4194304 totalling 12.00MiB
2019-12-13 18:15:13.035583: I tensorflow/core/common_runtime/bfc_allocator.cc:917] 1 Chunks of size 8388608 totalling 8.00MiB
2019-12-13 18:15:13.035595: I tensorflow/core/common_runtime/bfc_allocator.cc:917] 1 Chunks of size 17198848 totalling 16.40MiB
2019-12-13 18:15:13.035607: I tensorflow/core/common_runtime/bfc_allocator.cc:917] 1 Chunks of size 18874368 totalling 18.00MiB
2019-12-13 18:15:13.035620: I tensorflow/core/common_runtime/bfc_allocator.cc:921] Sum Total of in-use chunks: 74.38MiB
2019-12-13 18:15:13.035631: I tensorflow/core/common_runtime/bfc_allocator.cc:923] total_region_allocated_bytes_: 77987840 memory_limit_: 77987840 available bytes: 0 curr_region_allocation_bytes_: 155975680
2019-12-13 18:15:13.035648: I tensorflow/core/common_runtime/bfc_allocator.cc:929] Stats: 
Limit:                    77987840
InUse:                    77987840
MaxInUse:                 77987840
NumAllocs:                     112
MaxAllocSize:             18874368

2019-12-13 18:15:13.035679: W tensorflow/core/common_runtime/bfc_allocator.cc:424] *******************************************************************************************xxxxxxxxx
2019-12-13 18:15:23.042154: W tensorflow/core/common_runtime/bfc_allocator.cc:419] Allocator (GPU_0_bfc) ran out of memory trying to allocate 8B (rounded to 256).  Current allocation summary follows.
2019-12-13 18:15:23.042213: I tensorflow/core/common_runtime/bfc_allocator.cc:869] Bin (256): 	Total Chunks: 32, Chunks in use: 32. 8.0KiB allocated for chunks. 8.0KiB in use in bin. 1.1KiB client-requested in use in bin.
2019-12-13 18:15:23.042231: I tensorflow/core/common_runtime/bfc_allocator.cc:869] Bin (512): 	Total Chunks: 0, Chunks in use: 0. 0B allocated for chunks. 0B in use in bin. 0B client-requested in use in bin.
2019-12-13 18:15:23.042364: I tensorflow/core/common_runtime/bfc_allocator.cc:869] Bin (1024): 	Total Chunks: 25, Chunks in use: 25. 25.2KiB allocated for chunks. 25.2KiB in use in bin. 25.0KiB client-requested in use in bin.
2019-12-13 18:15:23.042379: I tensorflow/core/common_runtime/bfc_allocator.cc:869] Bin (2048): 	Total Chunks: 14, Chunks in use: 14. 28.0KiB allocated for chunks. 28.0KiB in use in bin. 28.0KiB client-requested in use in bin.
2019-12-13 18:15:23.042394: I tensorflow/core/common_runtime/bfc_allocator.cc:869] Bin (4096): 	Total Chunks: 10, Chunks in use: 10. 40.0KiB allocated for chunks. 40.0KiB in use in bin. 40.0KiB client-requested in use in bin.
2019-12-13 18:15:23.042410: I tensorflow/core/common_runtime/bfc_allocator.cc:869] Bin (8192): 	Total Chunks: 9, Chunks in use: 9. 72.0KiB allocated for chunks. 72.0KiB in use in bin. 72.0KiB client-requested in use in bin.
2019-12-13 18:15:23.042423: I tensorflow/core/common_runtime/bfc_allocator.cc:869] Bin (16384): 	Total Chunks: 0, Chunks in use: 0. 0B allocated for chunks. 0B in use in bin. 0B client-requested in use in bin.
2019-12-13 18:15:23.042440: I tensorflow/core/common_runtime/bfc_allocator.cc:869] Bin (32768): 	Total Chunks: 0, Chunks in use: 0. 0B allocated for chunks. 0B in use in bin. 0B client-requested in use in bin.
2019-12-13 18:15:23.042453: I tensorflow/core/common_runtime/bfc_allocator.cc:869] Bin (65536): 	Total Chunks: 0, Chunks in use: 0. 0B allocated for chunks. 0B in use in bin. 0B client-requested in use in bin.
2019-12-13 18:15:23.042469: I tensorflow/core/common_runtime/bfc_allocator.cc:869] Bin (131072): 	Total Chunks: 1, Chunks in use: 1. 243.0KiB allocated for chunks. 243.0KiB in use in bin. 243.0KiB client-requested in use in bin.
2019-12-13 18:15:23.042485: I tensorflow/core/common_runtime/bfc_allocator.cc:869] Bin (262144): 	Total Chunks: 1, Chunks in use: 1. 324.0KiB allocated for chunks. 324.0KiB in use in bin. 324.0KiB client-requested in use in bin.
2019-12-13 18:15:23.042501: I tensorflow/core/common_runtime/bfc_allocator.cc:869] Bin (524288): 	Total Chunks: 1, Chunks in use: 1. 512.0KiB allocated for chunks. 512.0KiB in use in bin. 512.0KiB client-requested in use in bin.
2019-12-13 18:15:23.042514: I tensorflow/core/common_runtime/bfc_allocator.cc:869] Bin (1048576): 	Total Chunks: 8, Chunks in use: 8. 8.00MiB allocated for chunks. 8.00MiB in use in bin. 8.00MiB client-requested in use in bin.
2019-12-13 18:15:23.042529: I tensorflow/core/common_runtime/bfc_allocator.cc:869] Bin (2097152): 	Total Chunks: 5, Chunks in use: 5. 10.75MiB allocated for chunks. 10.75MiB in use in bin. 10.75MiB client-requested in use in bin.
2019-12-13 18:15:23.042543: I tensorflow/core/common_runtime/bfc_allocator.cc:869] Bin (4194304): 	Total Chunks: 3, Chunks in use: 3. 12.00MiB allocated for chunks. 12.00MiB in use in bin. 12.00MiB client-requested in use in bin.
2019-12-13 18:15:23.042556: I tensorflow/core/common_runtime/bfc_allocator.cc:869] Bin (8388608): 	Total Chunks: 1, Chunks in use: 1. 8.00MiB allocated for chunks. 8.00MiB in use in bin. 8.00MiB client-requested in use in bin.
2019-12-13 18:15:23.042571: I tensorflow/core/common_runtime/bfc_allocator.cc:869] Bin (16777216): 	Total Chunks: 2, Chunks in use: 2. 34.40MiB allocated for chunks. 34.40MiB in use in bin. 27.00MiB client-requested in use in bin.
2019-12-13 18:15:23.042583: I tensorflow/core/common_runtime/bfc_allocator.cc:869] Bin (33554432): 	Total Chunks: 0, Chunks in use: 0. 0B allocated for chunks. 0B in use in bin. 0B client-requested in use in bin.
2019-12-13 18:15:23.042595: I tensorflow/core/common_runtime/bfc_allocator.cc:869] Bin (67108864): 	Total Chunks: 0, Chunks in use: 0. 0B allocated for chunks. 0B in use in bin. 0B client-requested in use in bin.
2019-12-13 18:15:23.042608: I tensorflow/core/common_runtime/bfc_allocator.cc:869] Bin (134217728): 	Total Chunks: 0, Chunks in use: 0. 0B allocated for chunks. 0B in use in bin. 0B client-requested in use in bin.
2019-12-13 18:15:23.042620: I tensorflow/core/common_runtime/bfc_allocator.cc:869] Bin (268435456): 	Total Chunks: 0, Chunks in use: 0. 0B allocated for chunks. 0B in use in bin. 0B client-requested in use in bin.
2019-12-13 18:15:23.042632: I tensorflow/core/common_runtime/bfc_allocator.cc:885] Bin for 256B was 256B, Chunk State: 
2019-12-13 18:15:23.042642: I tensorflow/core/common_runtime/bfc_allocator.cc:898] Next region of size 77987840
2019-12-13 18:15:23.042654: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa416000000 next 1 of size 1280
2019-12-13 18:15:23.042665: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa416000500 next 2 of size 256
2019-12-13 18:15:23.042674: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa416000600 next 3 of size 256
2019-12-13 18:15:23.042684: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa416000700 next 4 of size 256
2019-12-13 18:15:23.042695: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa416000800 next 5 of size 1024
2019-12-13 18:15:23.042704: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa416000c00 next 6 of size 256
2019-12-13 18:15:23.042714: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa416000d00 next 7 of size 256
2019-12-13 18:15:23.042724: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa416000e00 next 8 of size 256
2019-12-13 18:15:23.042733: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa416000f00 next 9 of size 256
2019-12-13 18:15:23.042743: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa416001000 next 10 of size 256
2019-12-13 18:15:23.042752: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa416001100 next 11 of size 256
2019-12-13 18:15:23.042762: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa416001200 next 12 of size 256
2019-12-13 18:15:23.042771: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa416001300 next 13 of size 256
2019-12-13 18:15:23.042781: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa416001400 next 14 of size 256
2019-12-13 18:15:23.042790: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa416001500 next 15 of size 524288
2019-12-13 18:15:23.042800: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa416081500 next 16 of size 18874368
2019-12-13 18:15:23.042810: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa417281500 next 17 of size 256
2019-12-13 18:15:23.042820: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa417281600 next 18 of size 2359296
2019-12-13 18:15:23.042831: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa4174c1600 next 19 of size 331776
2019-12-13 18:15:23.042841: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa417512600 next 20 of size 248832
2019-12-13 18:15:23.042852: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa41754f200 next 21 of size 2097152
2019-12-13 18:15:23.042862: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa41774f200 next 22 of size 1048576
2019-12-13 18:15:23.042872: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa41784f200 next 23 of size 256
2019-12-13 18:15:23.042881: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa41784f300 next 24 of size 256
2019-12-13 18:15:23.042891: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa41784f400 next 25 of size 256
2019-12-13 18:15:23.042901: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa41784f500 next 26 of size 256
2019-12-13 18:15:23.042910: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa41784f600 next 27 of size 256
2019-12-13 18:15:23.042920: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa41784f700 next 28 of size 256
2019-12-13 18:15:23.042929: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa41784f800 next 29 of size 256
2019-12-13 18:15:23.042939: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa41784f900 next 30 of size 256
2019-12-13 18:15:23.042948: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa41784fa00 next 31 of size 256
2019-12-13 18:15:23.042958: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa41784fb00 next 32 of size 256
2019-12-13 18:15:23.042968: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa41784fc00 next 33 of size 256
2019-12-13 18:15:23.042977: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa41784fd00 next 34 of size 256
2019-12-13 18:15:23.042986: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa41784fe00 next 35 of size 256
2019-12-13 18:15:23.042995: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa41784ff00 next 36 of size 256
2019-12-13 18:15:23.043005: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa417850000 next 37 of size 256
2019-12-13 18:15:23.043015: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa417850100 next 38 of size 256
2019-12-13 18:15:23.043024: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa417850200 next 39 of size 256
2019-12-13 18:15:23.043033: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa417850300 next 40 of size 256
2019-12-13 18:15:23.043042: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa417850400 next 41 of size 256
2019-12-13 18:15:23.043053: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa417850500 next 42 of size 4096
2019-12-13 18:15:23.043062: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa417851500 next 43 of size 4096
2019-12-13 18:15:23.043072: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa417852500 next 44 of size 1024
2019-12-13 18:15:23.043082: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa417852900 next 45 of size 1024
2019-12-13 18:15:23.043091: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa417852d00 next 46 of size 1024
2019-12-13 18:15:23.043101: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa417853100 next 47 of size 4096
2019-12-13 18:15:23.043110: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa417854100 next 48 of size 1048576
2019-12-13 18:15:23.043120: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa417954100 next 49 of size 4096
2019-12-13 18:15:23.043129: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa417955100 next 50 of size 1024
2019-12-13 18:15:23.043139: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa417955500 next 51 of size 1024
2019-12-13 18:15:23.043149: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa417955900 next 52 of size 1024
2019-12-13 18:15:23.043158: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa417955d00 next 53 of size 4096
2019-12-13 18:15:23.043168: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa417956d00 next 54 of size 4096
2019-12-13 18:15:23.043177: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa417957d00 next 55 of size 1048576
2019-12-13 18:15:23.043186: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa417a57d00 next 56 of size 1024
2019-12-13 18:15:23.043197: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa417a58100 next 57 of size 8192
2019-12-13 18:15:23.043206: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa417a5a100 next 58 of size 1024
2019-12-13 18:15:23.043216: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa417a5a500 next 59 of size 1048576
2019-12-13 18:15:23.043226: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa417b5a500 next 60 of size 2048
2019-12-13 18:15:23.043236: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa417b5ad00 next 61 of size 4194304
2019-12-13 18:15:23.043246: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa417f5ad00 next 62 of size 2048
2019-12-13 18:15:23.043256: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa417f5b500 next 63 of size 2048
2019-12-13 18:15:23.043265: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa417f5bd00 next 64 of size 2048
2019-12-13 18:15:23.043274: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa417f5c500 next 65 of size 4194304
2019-12-13 18:15:23.043284: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa41835c500 next 66 of size 2048
2019-12-13 18:15:23.043293: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa41835cd00 next 67 of size 4194304
2019-12-13 18:15:23.043303: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa41875cd00 next 68 of size 2048
2019-12-13 18:15:23.043313: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa41875d500 next 69 of size 1024
2019-12-13 18:15:23.043322: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa41875d900 next 70 of size 1024
2019-12-13 18:15:23.043332: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa41875dd00 next 71 of size 1024
2019-12-13 18:15:23.043342: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa41875e100 next 72 of size 2048
2019-12-13 18:15:23.043351: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa41875e900 next 73 of size 1024
2019-12-13 18:15:23.043360: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa41875ed00 next 74 of size 2048
2019-12-13 18:15:23.043370: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa41875f500 next 75 of size 2359296
2019-12-13 18:15:23.043379: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa41899f500 next 76 of size 4096
2019-12-13 18:15:23.043388: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa4189a0500 next 77 of size 2359296
2019-12-13 18:15:23.043398: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa418be0500 next 78 of size 1024
2019-12-13 18:15:23.043406: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa418be0900 next 79 of size 1024
2019-12-13 18:15:23.043415: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa418be0d00 next 80 of size 1048576
2019-12-13 18:15:23.043426: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa418ce0d00 next 81 of size 8388608
2019-12-13 18:15:23.043435: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa4194e0d00 next 82 of size 2048
2019-12-13 18:15:23.043444: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa4194e1500 next 83 of size 8192
2019-12-13 18:15:23.043454: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa4194e3500 next 84 of size 2048
2019-12-13 18:15:23.043464: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa4194e3d00 next 85 of size 8192
2019-12-13 18:15:23.043473: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa4194e5d00 next 86 of size 2048
2019-12-13 18:15:23.043483: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa4194e6500 next 87 of size 8192
2019-12-13 18:15:23.043492: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa4194e8500 next 88 of size 1024
2019-12-13 18:15:23.043501: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa4194e8900 next 89 of size 4096
2019-12-13 18:15:23.043511: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa4194e9900 next 90 of size 1024
2019-12-13 18:15:23.043520: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa4194e9d00 next 91 of size 2048
2019-12-13 18:15:23.043530: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa4194ea500 next 92 of size 8192
2019-12-13 18:15:23.043539: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa4194ec500 next 93 of size 2048
2019-12-13 18:15:23.043548: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa4194ecd00 next 94 of size 8192
2019-12-13 18:15:23.043558: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa4194eed00 next 95 of size 1024
2019-12-13 18:15:23.043567: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa4194ef100 next 96 of size 1024
2019-12-13 18:15:23.043577: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa4194ef500 next 97 of size 4096
2019-12-13 18:15:23.043586: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa4194f0500 next 98 of size 1024
2019-12-13 18:15:23.043596: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa4194f0900 next 99 of size 1048576
2019-12-13 18:15:23.043605: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa4195f0900 next 100 of size 2048
2019-12-13 18:15:23.043614: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa4195f1100 next 101 of size 1048576
2019-12-13 18:15:23.043624: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa4196f1100 next 102 of size 1024
2019-12-13 18:15:23.043633: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa4196f1500 next 103 of size 8192
2019-12-13 18:15:23.043643: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa4196f3500 next 104 of size 1024
2019-12-13 18:15:23.043652: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa4196f3900 next 105 of size 1024
2019-12-13 18:15:23.043662: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa4196f3d00 next 106 of size 4096
2019-12-13 18:15:23.043671: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa4196f4d00 next 107 of size 2097152
2019-12-13 18:15:23.043680: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa4198f4d00 next 108 of size 1048576
2019-12-13 18:15:23.043690: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa4199f4d00 next 109 of size 8192
2019-12-13 18:15:23.043699: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa4199f6d00 next 110 of size 1024
2019-12-13 18:15:23.043708: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa4199f7100 next 111 of size 8192
2019-12-13 18:15:23.043719: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa4199f9100 next 18446744073709551615 of size 17198848
2019-12-13 18:15:23.043728: I tensorflow/core/common_runtime/bfc_allocator.cc:914]      Summary of in-use Chunks by size: 
2019-12-13 18:15:23.043740: I tensorflow/core/common_runtime/bfc_allocator.cc:917] 32 Chunks of size 256 totalling 8.0KiB
2019-12-13 18:15:23.043750: I tensorflow/core/common_runtime/bfc_allocator.cc:917] 24 Chunks of size 1024 totalling 24.0KiB
2019-12-13 18:15:23.043761: I tensorflow/core/common_runtime/bfc_allocator.cc:917] 1 Chunks of size 1280 totalling 1.2KiB
2019-12-13 18:15:23.043771: I tensorflow/core/common_runtime/bfc_allocator.cc:917] 14 Chunks of size 2048 totalling 28.0KiB
2019-12-13 18:15:23.043781: I tensorflow/core/common_runtime/bfc_allocator.cc:917] 10 Chunks of size 4096 totalling 40.0KiB
2019-12-13 18:15:23.043792: I tensorflow/core/common_runtime/bfc_allocator.cc:917] 9 Chunks of size 8192 totalling 72.0KiB
2019-12-13 18:15:23.043803: I tensorflow/core/common_runtime/bfc_allocator.cc:917] 1 Chunks of size 248832 totalling 243.0KiB
2019-12-13 18:15:23.043822: I tensorflow/core/common_runtime/bfc_allocator.cc:917] 1 Chunks of size 331776 totalling 324.0KiB
2019-12-13 18:15:23.043833: I tensorflow/core/common_runtime/bfc_allocator.cc:917] 1 Chunks of size 524288 totalling 512.0KiB
2019-12-13 18:15:23.043844: I tensorflow/core/common_runtime/bfc_allocator.cc:917] 8 Chunks of size 1048576 totalling 8.00MiB
2019-12-13 18:15:23.043855: I tensorflow/core/common_runtime/bfc_allocator.cc:917] 2 Chunks of size 2097152 totalling 4.00MiB
2019-12-13 18:15:23.043866: I tensorflow/core/common_runtime/bfc_allocator.cc:917] 3 Chunks of size 2359296 totalling 6.75MiB
2019-12-13 18:15:23.043877: I tensorflow/core/common_runtime/bfc_allocator.cc:917] 3 Chunks of size 4194304 totalling 12.00MiB
2019-12-13 18:15:23.043888: I tensorflow/core/common_runtime/bfc_allocator.cc:917] 1 Chunks of size 8388608 totalling 8.00MiB
2019-12-13 18:15:23.043898: I tensorflow/core/common_runtime/bfc_allocator.cc:917] 1 Chunks of size 17198848 totalling 16.40MiB
2019-12-13 18:15:23.043909: I tensorflow/core/common_runtime/bfc_allocator.cc:917] 1 Chunks of size 18874368 totalling 18.00MiB
2019-12-13 18:15:23.043920: I tensorflow/core/common_runtime/bfc_allocator.cc:921] Sum Total of in-use chunks: 74.38MiB
2019-12-13 18:15:23.043930: I tensorflow/core/common_runtime/bfc_allocator.cc:923] total_region_allocated_bytes_: 77987840 memory_limit_: 77987840 available bytes: 0 curr_region_allocation_bytes_: 155975680
2019-12-13 18:15:23.043944: I tensorflow/core/common_runtime/bfc_allocator.cc:929] Stats: 
Limit:                    77987840
InUse:                    77987840
MaxInUse:                 77987840
NumAllocs:                     112
MaxAllocSize:             18874368

2019-12-13 18:15:23.043964: W tensorflow/core/common_runtime/bfc_allocator.cc:424] *******************************************************************************************xxxxxxxxx
2019-12-13 18:15:23.044013: W tensorflow/core/framework/op_kernel.cc:1651] OP_REQUIRES failed at resource_variable_ops.cc:556 : Resource exhausted: OOM when allocating tensor with shape[] and type int64 on /job:localhost/replica:0/task:0/device:GPU:0 by allocator GPU_0_bfc
2019-12-13 18:15:33.044520: W tensorflow/core/common_runtime/bfc_allocator.cc:419] Allocator (GPU_0_bfc) ran out of memory trying to allocate 4B (rounded to 256).  Current allocation summary follows.
2019-12-13 18:15:33.044575: I tensorflow/core/common_runtime/bfc_allocator.cc:869] Bin (256): 	Total Chunks: 32, Chunks in use: 32. 8.0KiB allocated for chunks. 8.0KiB in use in bin. 1.1KiB client-requested in use in bin.
2019-12-13 18:15:33.044591: I tensorflow/core/common_runtime/bfc_allocator.cc:869] Bin (512): 	Total Chunks: 0, Chunks in use: 0. 0B allocated for chunks. 0B in use in bin. 0B client-requested in use in bin.
2019-12-13 18:15:33.044607: I tensorflow/core/common_runtime/bfc_allocator.cc:869] Bin (1024): 	Total Chunks: 25, Chunks in use: 25. 25.2KiB allocated for chunks. 25.2KiB in use in bin. 25.0KiB client-requested in use in bin.
2019-12-13 18:15:33.044624: I tensorflow/core/common_runtime/bfc_allocator.cc:869] Bin (2048): 	Total Chunks: 14, Chunks in use: 14. 28.0KiB allocated for chunks. 28.0KiB in use in bin. 28.0KiB client-requested in use in bin.
2019-12-13 18:15:33.044639: I tensorflow/core/common_runtime/bfc_allocator.cc:869] Bin (4096): 	Total Chunks: 10, Chunks in use: 10. 40.0KiB allocated for chunks. 40.0KiB in use in bin. 40.0KiB client-requested in use in bin.
2019-12-13 18:15:33.044654: I tensorflow/core/common_runtime/bfc_allocator.cc:869] Bin (8192): 	Total Chunks: 9, Chunks in use: 9. 72.0KiB allocated for chunks. 72.0KiB in use in bin. 72.0KiB client-requested in use in bin.
2019-12-13 18:15:33.044667: I tensorflow/core/common_runtime/bfc_allocator.cc:869] Bin (16384): 	Total Chunks: 0, Chunks in use: 0. 0B allocated for chunks. 0B in use in bin. 0B client-requested in use in bin.
2019-12-13 18:15:33.044679: I tensorflow/core/common_runtime/bfc_allocator.cc:869] Bin (32768): 	Total Chunks: 0, Chunks in use: 0. 0B allocated for chunks. 0B in use in bin. 0B client-requested in use in bin.
2019-12-13 18:15:33.044691: I tensorflow/core/common_runtime/bfc_allocator.cc:869] Bin (65536): 	Total Chunks: 0, Chunks in use: 0. 0B allocated for chunks. 0B in use in bin. 0B client-requested in use in bin.
2019-12-13 18:15:33.044706: I tensorflow/core/common_runtime/bfc_allocator.cc:869] Bin (131072): 	Total Chunks: 1, Chunks in use: 1. 243.0KiB allocated for chunks. 243.0KiB in use in bin. 243.0KiB client-requested in use in bin.
2019-12-13 18:15:33.044720: I tensorflow/core/common_runtime/bfc_allocator.cc:869] Bin (262144): 	Total Chunks: 1, Chunks in use: 1. 324.0KiB allocated for chunks. 324.0KiB in use in bin. 324.0KiB client-requested in use in bin.
2019-12-13 18:15:33.044734: I tensorflow/core/common_runtime/bfc_allocator.cc:869] Bin (524288): 	Total Chunks: 1, Chunks in use: 1. 512.0KiB allocated for chunks. 512.0KiB in use in bin. 512.0KiB client-requested in use in bin.
2019-12-13 18:15:33.044749: I tensorflow/core/common_runtime/bfc_allocator.cc:869] Bin (1048576): 	Total Chunks: 8, Chunks in use: 8. 8.00MiB allocated for chunks. 8.00MiB in use in bin. 8.00MiB client-requested in use in bin.
2019-12-13 18:15:33.044763: I tensorflow/core/common_runtime/bfc_allocator.cc:869] Bin (2097152): 	Total Chunks: 5, Chunks in use: 5. 10.75MiB allocated for chunks. 10.75MiB in use in bin. 10.75MiB client-requested in use in bin.
2019-12-13 18:15:33.044777: I tensorflow/core/common_runtime/bfc_allocator.cc:869] Bin (4194304): 	Total Chunks: 3, Chunks in use: 3. 12.00MiB allocated for chunks. 12.00MiB in use in bin. 12.00MiB client-requested in use in bin.
2019-12-13 18:15:33.044791: I tensorflow/core/common_runtime/bfc_allocator.cc:869] Bin (8388608): 	Total Chunks: 1, Chunks in use: 1. 8.00MiB allocated for chunks. 8.00MiB in use in bin. 8.00MiB client-requested in use in bin.
2019-12-13 18:15:33.044805: I tensorflow/core/common_runtime/bfc_allocator.cc:869] Bin (16777216): 	Total Chunks: 2, Chunks in use: 2. 34.40MiB allocated for chunks. 34.40MiB in use in bin. 27.00MiB client-requested in use in bin.
2019-12-13 18:15:33.044818: I tensorflow/core/common_runtime/bfc_allocator.cc:869] Bin (33554432): 	Total Chunks: 0, Chunks in use: 0. 0B allocated for chunks. 0B in use in bin. 0B client-requested in use in bin.
2019-12-13 18:15:33.044830: I tensorflow/core/common_runtime/bfc_allocator.cc:869] Bin (67108864): 	Total Chunks: 0, Chunks in use: 0. 0B allocated for chunks. 0B in use in bin. 0B client-requested in use in bin.
2019-12-13 18:15:33.044842: I tensorflow/core/common_runtime/bfc_allocator.cc:869] Bin (134217728): 	Total Chunks: 0, Chunks in use: 0. 0B allocated for chunks. 0B in use in bin. 0B client-requested in use in bin.
2019-12-13 18:15:33.044853: I tensorflow/core/common_runtime/bfc_allocator.cc:869] Bin (268435456): 	Total Chunks: 0, Chunks in use: 0. 0B allocated for chunks. 0B in use in bin. 0B client-requested in use in bin.
2019-12-13 18:15:33.044865: I tensorflow/core/common_runtime/bfc_allocator.cc:885] Bin for 256B was 256B, Chunk State: 
2019-12-13 18:15:33.044874: I tensorflow/core/common_runtime/bfc_allocator.cc:898] Next region of size 77987840
2019-12-13 18:15:33.044887: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa416000000 next 1 of size 1280
2019-12-13 18:15:33.044897: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa416000500 next 2 of size 256
2019-12-13 18:15:33.044907: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa416000600 next 3 of size 256
2019-12-13 18:15:33.044916: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa416000700 next 4 of size 256
2019-12-13 18:15:33.044926: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa416000800 next 5 of size 1024
2019-12-13 18:15:33.044936: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa416000c00 next 6 of size 256
2019-12-13 18:15:33.044945: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa416000d00 next 7 of size 256
2019-12-13 18:15:33.044954: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa416000e00 next 8 of size 256
2019-12-13 18:15:33.044963: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa416000f00 next 9 of size 256
2019-12-13 18:15:33.044973: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa416001000 next 10 of size 256
2019-12-13 18:15:33.044982: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa416001100 next 11 of size 256
2019-12-13 18:15:33.044991: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa416001200 next 12 of size 256
2019-12-13 18:15:33.045000: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa416001300 next 13 of size 256
2019-12-13 18:15:33.045009: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa416001400 next 14 of size 256
2019-12-13 18:15:33.045019: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa416001500 next 15 of size 524288
2019-12-13 18:15:33.045029: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa416081500 next 16 of size 18874368
2019-12-13 18:15:33.045038: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa417281500 next 17 of size 256
2019-12-13 18:15:33.045047: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa417281600 next 18 of size 2359296
2019-12-13 18:15:33.045057: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa4174c1600 next 19 of size 331776
2019-12-13 18:15:33.045067: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa417512600 next 20 of size 248832
2019-12-13 18:15:33.045077: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa41754f200 next 21 of size 2097152
2019-12-13 18:15:33.045086: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa41774f200 next 22 of size 1048576
2019-12-13 18:15:33.045095: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa41784f200 next 23 of size 256
2019-12-13 18:15:33.045107: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa41784f300 next 24 of size 256
2019-12-13 18:15:33.045116: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa41784f400 next 25 of size 256
2019-12-13 18:15:33.045139: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa41784f500 next 26 of size 256
2019-12-13 18:15:33.045148: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa41784f600 next 27 of size 256
2019-12-13 18:15:33.045158: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa41784f700 next 28 of size 256
2019-12-13 18:15:33.045168: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa41784f800 next 29 of size 256
2019-12-13 18:15:33.045177: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa41784f900 next 30 of size 256
2019-12-13 18:15:33.045186: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa41784fa00 next 31 of size 256
2019-12-13 18:15:33.045195: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa41784fb00 next 32 of size 256
2019-12-13 18:15:33.045204: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa41784fc00 next 33 of size 256
2019-12-13 18:15:33.045213: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa41784fd00 next 34 of size 256
2019-12-13 18:15:33.045222: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa41784fe00 next 35 of size 256
2019-12-13 18:15:33.045231: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa41784ff00 next 36 of size 256
2019-12-13 18:15:33.045240: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa417850000 next 37 of size 256
2019-12-13 18:15:33.045250: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa417850100 next 38 of size 256
2019-12-13 18:15:33.045259: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa417850200 next 39 of size 256
2019-12-13 18:15:33.045268: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa417850300 next 40 of size 256
2019-12-13 18:15:33.045278: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa417850400 next 41 of size 256
2019-12-13 18:15:33.045290: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa417850500 next 42 of size 4096
2019-12-13 18:15:33.045301: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa417851500 next 43 of size 4096
2019-12-13 18:15:33.045310: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa417852500 next 44 of size 1024
2019-12-13 18:15:33.045320: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa417852900 next 45 of size 1024
2019-12-13 18:15:33.045329: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa417852d00 next 46 of size 1024
2019-12-13 18:15:33.045338: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa417853100 next 47 of size 4096
2019-12-13 18:15:33.045347: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa417854100 next 48 of size 1048576
2019-12-13 18:15:33.045357: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa417954100 next 49 of size 4096
2019-12-13 18:15:33.045366: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa417955100 next 50 of size 1024
2019-12-13 18:15:33.045375: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa417955500 next 51 of size 1024
2019-12-13 18:15:33.045384: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa417955900 next 52 of size 1024
2019-12-13 18:15:33.045393: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa417955d00 next 53 of size 4096
2019-12-13 18:15:33.045403: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa417956d00 next 54 of size 4096
2019-12-13 18:15:33.045412: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa417957d00 next 55 of size 1048576
2019-12-13 18:15:33.045421: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa417a57d00 next 56 of size 1024
2019-12-13 18:15:33.045430: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa417a58100 next 57 of size 8192
2019-12-13 18:15:33.045441: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa417a5a100 next 58 of size 1024
2019-12-13 18:15:33.045451: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa417a5a500 next 59 of size 1048576
2019-12-13 18:15:33.045461: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa417b5a500 next 60 of size 2048
2019-12-13 18:15:33.045471: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa417b5ad00 next 61 of size 4194304
2019-12-13 18:15:33.045481: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa417f5ad00 next 62 of size 2048
2019-12-13 18:15:33.045490: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa417f5b500 next 63 of size 2048
2019-12-13 18:15:33.045499: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa417f5bd00 next 64 of size 2048
2019-12-13 18:15:33.045509: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa417f5c500 next 65 of size 4194304
2019-12-13 18:15:33.045518: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa41835c500 next 66 of size 2048
2019-12-13 18:15:33.045527: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa41835cd00 next 67 of size 4194304
2019-12-13 18:15:33.045536: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa41875cd00 next 68 of size 2048
2019-12-13 18:15:33.045545: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa41875d500 next 69 of size 1024
2019-12-13 18:15:33.045554: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa41875d900 next 70 of size 1024
2019-12-13 18:15:33.045563: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa41875dd00 next 71 of size 1024
2019-12-13 18:15:33.045572: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa41875e100 next 72 of size 2048
2019-12-13 18:15:33.045582: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa41875e900 next 73 of size 1024
2019-12-13 18:15:33.045591: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa41875ed00 next 74 of size 2048
2019-12-13 18:15:33.045600: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa41875f500 next 75 of size 2359296
2019-12-13 18:15:33.045609: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa41899f500 next 76 of size 4096
2019-12-13 18:15:33.045618: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa4189a0500 next 77 of size 2359296
2019-12-13 18:15:33.045628: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa418be0500 next 78 of size 1024
2019-12-13 18:15:33.045638: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa418be0900 next 79 of size 1024
2019-12-13 18:15:33.045647: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa418be0d00 next 80 of size 1048576
2019-12-13 18:15:33.045657: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa418ce0d00 next 81 of size 8388608
2019-12-13 18:15:33.045666: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa4194e0d00 next 82 of size 2048
2019-12-13 18:15:33.045675: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa4194e1500 next 83 of size 8192
2019-12-13 18:15:33.045684: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa4194e3500 next 84 of size 2048
2019-12-13 18:15:33.045693: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa4194e3d00 next 85 of size 8192
2019-12-13 18:15:33.045703: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa4194e5d00 next 86 of size 2048
2019-12-13 18:15:33.045712: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa4194e6500 next 87 of size 8192
2019-12-13 18:15:33.045721: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa4194e8500 next 88 of size 1024
2019-12-13 18:15:33.045730: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa4194e8900 next 89 of size 4096
2019-12-13 18:15:33.045739: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa4194e9900 next 90 of size 1024
2019-12-13 18:15:33.045748: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa4194e9d00 next 91 of size 2048
2019-12-13 18:15:33.045758: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa4194ea500 next 92 of size 8192
2019-12-13 18:15:33.045767: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa4194ec500 next 93 of size 2048
2019-12-13 18:15:33.045776: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa4194ecd00 next 94 of size 8192
2019-12-13 18:15:33.045785: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa4194eed00 next 95 of size 1024
2019-12-13 18:15:33.045794: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa4194ef100 next 96 of size 1024
2019-12-13 18:15:33.045802: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa4194ef500 next 97 of size 4096
2019-12-13 18:15:33.045812: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa4194f0500 next 98 of size 1024
2019-12-13 18:15:33.045821: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa4194f0900 next 99 of size 1048576
2019-12-13 18:15:33.045830: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa4195f0900 next 100 of size 2048
2019-12-13 18:15:33.045839: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa4195f1100 next 101 of size 1048576
2019-12-13 18:15:33.045848: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa4196f1100 next 102 of size 1024
2019-12-13 18:15:33.045857: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa4196f1500 next 103 of size 8192
2019-12-13 18:15:33.045866: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa4196f3500 next 104 of size 1024
2019-12-13 18:15:33.045875: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa4196f3900 next 105 of size 1024
2019-12-13 18:15:33.045884: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa4196f3d00 next 106 of size 4096
2019-12-13 18:15:33.045893: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa4196f4d00 next 107 of size 2097152
2019-12-13 18:15:33.045902: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa4198f4d00 next 108 of size 1048576
2019-12-13 18:15:33.045911: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa4199f4d00 next 109 of size 8192
2019-12-13 18:15:33.045920: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa4199f6d00 next 110 of size 1024
2019-12-13 18:15:33.045929: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa4199f7100 next 111 of size 8192
2019-12-13 18:15:33.045939: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa4199f9100 next 18446744073709551615 of size 17198848
2019-12-13 18:15:33.045947: I tensorflow/core/common_runtime/bfc_allocator.cc:914]      Summary of in-use Chunks by size: 
2019-12-13 18:15:33.045959: I tensorflow/core/common_runtime/bfc_allocator.cc:917] 32 Chunks of size 256 totalling 8.0KiB
2019-12-13 18:15:33.045970: I tensorflow/core/common_runtime/bfc_allocator.cc:917] 24 Chunks of size 1024 totalling 24.0KiB
2019-12-13 18:15:33.045980: I tensorflow/core/common_runtime/bfc_allocator.cc:917] 1 Chunks of size 1280 totalling 1.2KiB
2019-12-13 18:15:33.045989: I tensorflow/core/common_runtime/bfc_allocator.cc:917] 14 Chunks of size 2048 totalling 28.0KiB
2019-12-13 18:15:33.045999: I tensorflow/core/common_runtime/bfc_allocator.cc:917] 10 Chunks of size 4096 totalling 40.0KiB
2019-12-13 18:15:33.046009: I tensorflow/core/common_runtime/bfc_allocator.cc:917] 9 Chunks of size 8192 totalling 72.0KiB
2019-12-13 18:15:33.046020: I tensorflow/core/common_runtime/bfc_allocator.cc:917] 1 Chunks of size 248832 totalling 243.0KiB
2019-12-13 18:15:33.046030: I tensorflow/core/common_runtime/bfc_allocator.cc:917] 1 Chunks of size 331776 totalling 324.0KiB
2019-12-13 18:15:33.046040: I tensorflow/core/common_runtime/bfc_allocator.cc:917] 1 Chunks of size 524288 totalling 512.0KiB
2019-12-13 18:15:33.046051: I tensorflow/core/common_runtime/bfc_allocator.cc:917] 8 Chunks of size 1048576 totalling 8.00MiB
2019-12-13 18:15:33.046061: I tensorflow/core/common_runtime/bfc_allocator.cc:917] 2 Chunks of size 2097152 totalling 4.00MiB
2019-12-13 18:15:33.046070: I tensorflow/core/common_runtime/bfc_allocator.cc:917] 3 Chunks of size 2359296 totalling 6.75MiB
2019-12-13 18:15:33.046081: I tensorflow/core/common_runtime/bfc_allocator.cc:917] 3 Chunks of size 4194304 totalling 12.00MiB
2019-12-13 18:15:33.046091: I tensorflow/core/common_runtime/bfc_allocator.cc:917] 1 Chunks of size 8388608 totalling 8.00MiB
2019-12-13 18:15:33.046101: I tensorflow/core/common_runtime/bfc_allocator.cc:917] 1 Chunks of size 17198848 totalling 16.40MiB
2019-12-13 18:15:33.046111: I tensorflow/core/common_runtime/bfc_allocator.cc:917] 1 Chunks of size 18874368 totalling 18.00MiB
2019-12-13 18:15:33.046121: I tensorflow/core/common_runtime/bfc_allocator.cc:921] Sum Total of in-use chunks: 74.38MiB
2019-12-13 18:15:33.046130: I tensorflow/core/common_runtime/bfc_allocator.cc:923] total_region_allocated_bytes_: 77987840 memory_limit_: 77987840 available bytes: 0 curr_region_allocation_bytes_: 155975680
2019-12-13 18:15:33.046145: I tensorflow/core/common_runtime/bfc_allocator.cc:929] Stats: 
Limit:                    77987840
InUse:                    77987840
MaxInUse:                 77987840
NumAllocs:                     112
MaxAllocSize:             18874368

2019-12-13 18:15:33.046170: W tensorflow/core/common_runtime/bfc_allocator.cc:424] *******************************************************************************************xxxxxxxxx
2019-12-13 18:15:33.046199: W tensorflow/core/framework/op_kernel.cc:1651] OP_REQUIRES failed at cast_op.cc:109 : Resource exhausted: OOM when allocating tensor with shape[] and type float on /job:localhost/replica:0/task:0/device:GPU:0 by allocator GPU_0_bfc
2019-12-13 18:15:43.046601: W tensorflow/core/common_runtime/bfc_allocator.cc:419] Allocator (GPU_0_bfc) ran out of memory trying to allocate 4B (rounded to 256).  Current allocation summary follows.
2019-12-13 18:15:43.046665: I tensorflow/core/common_runtime/bfc_allocator.cc:869] Bin (256): 	Total Chunks: 32, Chunks in use: 32. 8.0KiB allocated for chunks. 8.0KiB in use in bin. 1.1KiB client-requested in use in bin.
2019-12-13 18:15:43.046680: I tensorflow/core/common_runtime/bfc_allocator.cc:869] Bin (512): 	Total Chunks: 0, Chunks in use: 0. 0B allocated for chunks. 0B in use in bin. 0B client-requested in use in bin.
2019-12-13 18:15:43.046711: I tensorflow/core/common_runtime/bfc_allocator.cc:869] Bin (1024): 	Total Chunks: 25, Chunks in use: 25. 25.2KiB allocated for chunks. 25.2KiB in use in bin. 25.0KiB client-requested in use in bin.
2019-12-13 18:15:43.046725: I tensorflow/core/common_runtime/bfc_allocator.cc:869] Bin (2048): 	Total Chunks: 14, Chunks in use: 14. 28.0KiB allocated for chunks. 28.0KiB in use in bin. 28.0KiB client-requested in use in bin.
2019-12-13 18:15:43.046814: I tensorflow/core/common_runtime/bfc_allocator.cc:869] Bin (4096): 	Total Chunks: 10, Chunks in use: 10. 40.0KiB allocated for chunks. 40.0KiB in use in bin. 40.0KiB client-requested in use in bin.
2019-12-13 18:15:43.046829: I tensorflow/core/common_runtime/bfc_allocator.cc:869] Bin (8192): 	Total Chunks: 9, Chunks in use: 9. 72.0KiB allocated for chunks. 72.0KiB in use in bin. 72.0KiB client-requested in use in bin.
2019-12-13 18:15:43.046842: I tensorflow/core/common_runtime/bfc_allocator.cc:869] Bin (16384): 	Total Chunks: 0, Chunks in use: 0. 0B allocated for chunks. 0B in use in bin. 0B client-requested in use in bin.
2019-12-13 18:15:43.046854: I tensorflow/core/common_runtime/bfc_allocator.cc:869] Bin (32768): 	Total Chunks: 0, Chunks in use: 0. 0B allocated for chunks. 0B in use in bin. 0B client-requested in use in bin.
2019-12-13 18:15:43.046866: I tensorflow/core/common_runtime/bfc_allocator.cc:869] Bin (65536): 	Total Chunks: 0, Chunks in use: 0. 0B allocated for chunks. 0B in use in bin. 0B client-requested in use in bin.
2019-12-13 18:15:43.046881: I tensorflow/core/common_runtime/bfc_allocator.cc:869] Bin (131072): 	Total Chunks: 1, Chunks in use: 1. 243.0KiB allocated for chunks. 243.0KiB in use in bin. 243.0KiB client-requested in use in bin.
2019-12-13 18:15:43.046896: I tensorflow/core/common_runtime/bfc_allocator.cc:869] Bin (262144): 	Total Chunks: 1, Chunks in use: 1. 324.0KiB allocated for chunks. 324.0KiB in use in bin. 324.0KiB client-requested in use in bin.
2019-12-13 18:15:43.046910: I tensorflow/core/common_runtime/bfc_allocator.cc:869] Bin (524288): 	Total Chunks: 1, Chunks in use: 1. 512.0KiB allocated for chunks. 512.0KiB in use in bin. 512.0KiB client-requested in use in bin.
2019-12-13 18:15:43.046924: I tensorflow/core/common_runtime/bfc_allocator.cc:869] Bin (1048576): 	Total Chunks: 8, Chunks in use: 8. 8.00MiB allocated for chunks. 8.00MiB in use in bin. 8.00MiB client-requested in use in bin.
2019-12-13 18:15:43.046938: I tensorflow/core/common_runtime/bfc_allocator.cc:869] Bin (2097152): 	Total Chunks: 5, Chunks in use: 5. 10.75MiB allocated for chunks. 10.75MiB in use in bin. 10.75MiB client-requested in use in bin.
2019-12-13 18:15:43.046952: I tensorflow/core/common_runtime/bfc_allocator.cc:869] Bin (4194304): 	Total Chunks: 3, Chunks in use: 3. 12.00MiB allocated for chunks. 12.00MiB in use in bin. 12.00MiB client-requested in use in bin.
2019-12-13 18:15:43.046966: I tensorflow/core/common_runtime/bfc_allocator.cc:869] Bin (8388608): 	Total Chunks: 1, Chunks in use: 1. 8.00MiB allocated for chunks. 8.00MiB in use in bin. 8.00MiB client-requested in use in bin.
2019-12-13 18:15:43.046980: I tensorflow/core/common_runtime/bfc_allocator.cc:869] Bin (16777216): 	Total Chunks: 2, Chunks in use: 2. 34.40MiB allocated for chunks. 34.40MiB in use in bin. 27.00MiB client-requested in use in bin.
2019-12-13 18:15:43.046992: I tensorflow/core/common_runtime/bfc_allocator.cc:869] Bin (33554432): 	Total Chunks: 0, Chunks in use: 0. 0B allocated for chunks. 0B in use in bin. 0B client-requested in use in bin.
2019-12-13 18:15:43.047003: I tensorflow/core/common_runtime/bfc_allocator.cc:869] Bin (67108864): 	Total Chunks: 0, Chunks in use: 0. 0B allocated for chunks. 0B in use in bin. 0B client-requested in use in bin.
2019-12-13 18:15:43.047015: I tensorflow/core/common_runtime/bfc_allocator.cc:869] Bin (134217728): 	Total Chunks: 0, Chunks in use: 0. 0B allocated for chunks. 0B in use in bin. 0B client-requested in use in bin.
2019-12-13 18:15:43.047027: I tensorflow/core/common_runtime/bfc_allocator.cc:869] Bin (268435456): 	Total Chunks: 0, Chunks in use: 0. 0B allocated for chunks. 0B in use in bin. 0B client-requested in use in bin.
2019-12-13 18:15:43.047039: I tensorflow/core/common_runtime/bfc_allocator.cc:885] Bin for 256B was 256B, Chunk State: 
2019-12-13 18:15:43.047048: I tensorflow/core/common_runtime/bfc_allocator.cc:898] Next region of size 77987840
2019-12-13 18:15:43.047061: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa416000000 next 1 of size 1280
2019-12-13 18:15:43.047071: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa416000500 next 2 of size 256
2019-12-13 18:15:43.047080: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa416000600 next 3 of size 256
2019-12-13 18:15:43.047089: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa416000700 next 4 of size 256
2019-12-13 18:15:43.047100: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa416000800 next 5 of size 1024
2019-12-13 18:15:43.047109: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa416000c00 next 6 of size 256
2019-12-13 18:15:43.047118: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa416000d00 next 7 of size 256
2019-12-13 18:15:43.047127: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa416000e00 next 8 of size 256
2019-12-13 18:15:43.047136: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa416000f00 next 9 of size 256
2019-12-13 18:15:43.047146: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa416001000 next 10 of size 256
2019-12-13 18:15:43.047155: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa416001100 next 11 of size 256
2019-12-13 18:15:43.047164: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa416001200 next 12 of size 256
2019-12-13 18:15:43.047173: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa416001300 next 13 of size 256
2019-12-13 18:15:43.047182: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa416001400 next 14 of size 256
2019-12-13 18:15:43.047191: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa416001500 next 15 of size 524288
2019-12-13 18:15:43.047201: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa416081500 next 16 of size 18874368
2019-12-13 18:15:43.047210: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa417281500 next 17 of size 256
2019-12-13 18:15:43.047219: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa417281600 next 18 of size 2359296
2019-12-13 18:15:43.047229: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa4174c1600 next 19 of size 331776
2019-12-13 18:15:43.047239: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa417512600 next 20 of size 248832
2019-12-13 18:15:43.047249: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa41754f200 next 21 of size 2097152
2019-12-13 18:15:43.047258: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa41774f200 next 22 of size 1048576
2019-12-13 18:15:43.047267: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa41784f200 next 23 of size 256
2019-12-13 18:15:43.047276: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa41784f300 next 24 of size 256
2019-12-13 18:15:43.047285: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa41784f400 next 25 of size 256
2019-12-13 18:15:43.047294: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa41784f500 next 26 of size 256
2019-12-13 18:15:43.047303: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa41784f600 next 27 of size 256
2019-12-13 18:15:43.047312: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa41784f700 next 28 of size 256
2019-12-13 18:15:43.047321: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa41784f800 next 29 of size 256
2019-12-13 18:15:43.047330: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa41784f900 next 30 of size 256
2019-12-13 18:15:43.047339: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa41784fa00 next 31 of size 256
2019-12-13 18:15:43.047347: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa41784fb00 next 32 of size 256
2019-12-13 18:15:43.047356: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa41784fc00 next 33 of size 256
2019-12-13 18:15:43.047365: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa41784fd00 next 34 of size 256
2019-12-13 18:15:43.047374: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa41784fe00 next 35 of size 256
2019-12-13 18:15:43.047383: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa41784ff00 next 36 of size 256
2019-12-13 18:15:43.047392: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa417850000 next 37 of size 256
2019-12-13 18:15:43.047401: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa417850100 next 38 of size 256
2019-12-13 18:15:43.047410: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa417850200 next 39 of size 256
2019-12-13 18:15:43.047419: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa417850300 next 40 of size 256
2019-12-13 18:15:43.047429: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa417850400 next 41 of size 256
2019-12-13 18:15:43.047438: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa417850500 next 42 of size 4096
2019-12-13 18:15:43.047447: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa417851500 next 43 of size 4096
2019-12-13 18:15:43.047457: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa417852500 next 44 of size 1024
2019-12-13 18:15:43.047466: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa417852900 next 45 of size 1024
2019-12-13 18:15:43.047475: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa417852d00 next 46 of size 1024
2019-12-13 18:15:43.047484: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa417853100 next 47 of size 4096
2019-12-13 18:15:43.047493: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa417854100 next 48 of size 1048576
2019-12-13 18:15:43.047502: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa417954100 next 49 of size 4096
2019-12-13 18:15:43.047511: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa417955100 next 50 of size 1024
2019-12-13 18:15:43.047520: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa417955500 next 51 of size 1024
2019-12-13 18:15:43.047529: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa417955900 next 52 of size 1024
2019-12-13 18:15:43.047538: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa417955d00 next 53 of size 4096
2019-12-13 18:15:43.047547: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa417956d00 next 54 of size 4096
2019-12-13 18:15:43.047556: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa417957d00 next 55 of size 1048576
2019-12-13 18:15:43.047565: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa417a57d00 next 56 of size 1024
2019-12-13 18:15:43.047575: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa417a58100 next 57 of size 8192
2019-12-13 18:15:43.047585: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa417a5a100 next 58 of size 1024
2019-12-13 18:15:43.047594: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa417a5a500 next 59 of size 1048576
2019-12-13 18:15:43.047603: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa417b5a500 next 60 of size 2048
2019-12-13 18:15:43.047613: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa417b5ad00 next 61 of size 4194304
2019-12-13 18:15:43.047622: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa417f5ad00 next 62 of size 2048
2019-12-13 18:15:43.047631: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa417f5b500 next 63 of size 2048
2019-12-13 18:15:43.047640: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa417f5bd00 next 64 of size 2048
2019-12-13 18:15:43.047649: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa417f5c500 next 65 of size 4194304
2019-12-13 18:15:43.047658: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa41835c500 next 66 of size 2048
2019-12-13 18:15:43.047667: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa41835cd00 next 67 of size 4194304
2019-12-13 18:15:43.047676: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa41875cd00 next 68 of size 2048
2019-12-13 18:15:43.047685: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa41875d500 next 69 of size 1024
2019-12-13 18:15:43.047694: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa41875d900 next 70 of size 1024
2019-12-13 18:15:43.047703: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa41875dd00 next 71 of size 1024
2019-12-13 18:15:43.047712: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa41875e100 next 72 of size 2048
2019-12-13 18:15:43.047721: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa41875e900 next 73 of size 1024
2019-12-13 18:15:43.047730: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa41875ed00 next 74 of size 2048
2019-12-13 18:15:43.047739: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa41875f500 next 75 of size 2359296
2019-12-13 18:15:43.047748: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa41899f500 next 76 of size 4096
2019-12-13 18:15:43.047757: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa4189a0500 next 77 of size 2359296
2019-12-13 18:15:43.047766: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa418be0500 next 78 of size 1024
2019-12-13 18:15:43.047775: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa418be0900 next 79 of size 1024
2019-12-13 18:15:43.047784: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa418be0d00 next 80 of size 1048576
2019-12-13 18:15:43.047793: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa418ce0d00 next 81 of size 8388608
2019-12-13 18:15:43.047802: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa4194e0d00 next 82 of size 2048
2019-12-13 18:15:43.047811: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa4194e1500 next 83 of size 8192
2019-12-13 18:15:43.047821: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa4194e3500 next 84 of size 2048
2019-12-13 18:15:43.047830: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa4194e3d00 next 85 of size 8192
2019-12-13 18:15:43.047838: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa4194e5d00 next 86 of size 2048
2019-12-13 18:15:43.047848: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa4194e6500 next 87 of size 8192
2019-12-13 18:15:43.047857: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa4194e8500 next 88 of size 1024
2019-12-13 18:15:43.047865: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa4194e8900 next 89 of size 4096
2019-12-13 18:15:43.047875: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa4194e9900 next 90 of size 1024
2019-12-13 18:15:43.047883: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa4194e9d00 next 91 of size 2048
2019-12-13 18:15:43.047893: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa4194ea500 next 92 of size 8192
2019-12-13 18:15:43.047902: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa4194ec500 next 93 of size 2048
2019-12-13 18:15:43.047911: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa4194ecd00 next 94 of size 8192
2019-12-13 18:15:43.047920: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa4194eed00 next 95 of size 1024
2019-12-13 18:15:43.047929: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa4194ef100 next 96 of size 1024
2019-12-13 18:15:43.047937: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa4194ef500 next 97 of size 4096
2019-12-13 18:15:43.047946: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa4194f0500 next 98 of size 1024
2019-12-13 18:15:43.047955: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa4194f0900 next 99 of size 1048576
2019-12-13 18:15:43.047964: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa4195f0900 next 100 of size 2048
2019-12-13 18:15:43.047973: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa4195f1100 next 101 of size 1048576
2019-12-13 18:15:43.047982: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa4196f1100 next 102 of size 1024
2019-12-13 18:15:43.047991: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa4196f1500 next 103 of size 8192
2019-12-13 18:15:43.048000: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa4196f3500 next 104 of size 1024
2019-12-13 18:15:43.048009: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa4196f3900 next 105 of size 1024
2019-12-13 18:15:43.048019: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa4196f3d00 next 106 of size 4096
2019-12-13 18:15:43.048028: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa4196f4d00 next 107 of size 2097152
2019-12-13 18:15:43.048037: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa4198f4d00 next 108 of size 1048576
2019-12-13 18:15:43.048046: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa4199f4d00 next 109 of size 8192
2019-12-13 18:15:43.048055: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa4199f6d00 next 110 of size 1024
2019-12-13 18:15:43.048064: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa4199f7100 next 111 of size 8192
2019-12-13 18:15:43.048073: I tensorflow/core/common_runtime/bfc_allocator.cc:905] InUse at 0x7fa4199f9100 next 18446744073709551615 of size 17198848
2019-12-13 18:15:43.048082: I tensorflow/core/common_runtime/bfc_allocator.cc:914]      Summary of in-use Chunks by size: 
2019-12-13 18:15:43.048094: I tensorflow/core/common_runtime/bfc_allocator.cc:917] 32 Chunks of size 256 totalling 8.0KiB
2019-12-13 18:15:43.048104: I tensorflow/core/common_runtime/bfc_allocator.cc:917] 24 Chunks of size 1024 totalling 24.0KiB
2019-12-13 18:15:43.048114: I tensorflow/core/common_runtime/bfc_allocator.cc:917] 1 Chunks of size 1280 totalling 1.2KiB
2019-12-13 18:15:43.048124: I tensorflow/core/common_runtime/bfc_allocator.cc:917] 14 Chunks of size 2048 totalling 28.0KiB
2019-12-13 18:15:43.048134: I tensorflow/core/common_runtime/bfc_allocator.cc:917] 10 Chunks of size 4096 totalling 40.0KiB
2019-12-13 18:15:43.048144: I tensorflow/core/common_runtime/bfc_allocator.cc:917] 9 Chunks of size 8192 totalling 72.0KiB
2019-12-13 18:15:43.048155: I tensorflow/core/common_runtime/bfc_allocator.cc:917] 1 Chunks of size 248832 totalling 243.0KiB
2019-12-13 18:15:43.048165: I tensorflow/core/common_runtime/bfc_allocator.cc:917] 1 Chunks of size 331776 totalling 324.0KiB
2019-12-13 18:15:43.048176: I tensorflow/core/common_runtime/bfc_allocator.cc:917] 1 Chunks of size 524288 totalling 512.0KiB
2019-12-13 18:15:43.048186: I tensorflow/core/common_runtime/bfc_allocator.cc:917] 8 Chunks of size 1048576 totalling 8.00MiB
2019-12-13 18:15:43.048196: I tensorflow/core/common_runtime/bfc_allocator.cc:917] 2 Chunks of size 2097152 totalling 4.00MiB
2019-12-13 18:15:43.048206: I tensorflow/core/common_runtime/bfc_allocator.cc:917] 3 Chunks of size 2359296 totalling 6.75MiB
2019-12-13 18:15:43.048216: I tensorflow/core/common_runtime/bfc_allocator.cc:917] 3 Chunks of size 4194304 totalling 12.00MiB
2019-12-13 18:15:43.048226: I tensorflow/core/common_runtime/bfc_allocator.cc:917] 1 Chunks of size 8388608 totalling 8.00MiB
2019-12-13 18:15:43.048236: I tensorflow/core/common_runtime/bfc_allocator.cc:917] 1 Chunks of size 17198848 totalling 16.40MiB
2019-12-13 18:15:43.048247: I tensorflow/core/common_runtime/bfc_allocator.cc:917] 1 Chunks of size 18874368 totalling 18.00MiB
2019-12-13 18:15:43.048257: I tensorflow/core/common_runtime/bfc_allocator.cc:921] Sum Total of in-use chunks: 74.38MiB
2019-12-13 18:15:43.048266: I tensorflow/core/common_runtime/bfc_allocator.cc:923] total_region_allocated_bytes_: 77987840 memory_limit_: 77987840 available bytes: 0 curr_region_allocation_bytes_: 155975680
2019-12-13 18:15:43.048280: I tensorflow/core/common_runtime/bfc_allocator.cc:929] Stats: 
Limit:                    77987840
InUse:                    77987840
MaxInUse:                 77987840
NumAllocs:                     112
MaxAllocSize:             18874368

2019-12-13 18:15:43.048303: W tensorflow/core/common_runtime/bfc_allocator.cc:424] *******************************************************************************************xxxxxxxxx
2019-12-13 18:15:43.048335: W tensorflow/core/framework/op_kernel.cc:1651] OP_REQUIRES failed at cwise_ops_common.cc:82 : Resource exhausted: OOM when allocating tensor with shape[] and type float on /job:localhost/replica:0/task:0/device:GPU:0 by allocator GPU_0_bfc
^C

Test RetinaNet model after trained

In [24]:
# adjust this to point to your downloaded/trained model
# models can be downloaded here: https://github.com/fizyr/keras-retinanet/releases
model_path_t = os.path.join('.', 'snapshots', 'resnet50_csv_05_convert.h5')

# load retinanet model
model_t = models.load_model(model_path_t, backbone_name='resnet50')

# if the model is not converted to an inference model, use the line below
# see: https://github.com/fizyr/keras-retinanet#converting-a-training-model-to-inference-model
#model = models.convert_model(model)

#print(model_t.summary())

# load label to names mapping for visualization purposes

# load label to names mapping for visualization purposes
labels_to_names = {0: 'Platelets', 1: 'WBC', 2: 'RBC'}
tracking <tf.Variable 'Variable_10:0' shape=(9, 4) dtype=float32> anchors
tracking <tf.Variable 'Variable_11:0' shape=(9, 4) dtype=float32> anchors
tracking <tf.Variable 'Variable_12:0' shape=(9, 4) dtype=float32> anchors
tracking <tf.Variable 'Variable_13:0' shape=(9, 4) dtype=float32> anchors
tracking <tf.Variable 'Variable_14:0' shape=(9, 4) dtype=float32> anchors

Run detection on example

In [25]:
# load image
image = read_image_bgr('./eval_images/2020-04-16_15-58-40_CU.jpg')

# copy to draw on
draw = image.copy()
draw = cv2.cvtColor(draw, cv2.COLOR_BGR2RGB)

# preprocess image for network
image = preprocess_image(image)
image, scale = resize_image(image)

# process image
start = time.time()
boxes, scores, labels = model_t.predict_on_batch(np.expand_dims(image, axis=0))
print("processing time: ", time.time() - start)

# correct for image scale
boxes /= scale

# visualize detections
for box, score, label in zip(boxes[0], scores[0], labels[0]):
    # scores are sorted so we can break
    if score < 0.5:
        break
        
    color = label_color(label)
    
    b = box.astype(int)
    draw_box(draw, b, color=color)
    
    caption = "{} {:.3f}".format(labels_to_names[label], score)
    draw_caption(draw, b, caption)
    
plt.figure(figsize=(15, 15))
plt.axis('on')
plt.imshow(draw)
plt.show()
processing time:  0.7381088733673096
In [26]:
# load image
image = read_image_bgr('./eval_images/2020-04-18_16-06-00_CU.jpg')

# copy to draw on
draw = image.copy()
draw = cv2.cvtColor(draw, cv2.COLOR_BGR2RGB)

# preprocess image for network
image = preprocess_image(image)
image, scale = resize_image(image)

# process image
start = time.time()
boxes, scores, labels = model_t.predict_on_batch(np.expand_dims(image, axis=0))
print("processing time: ", time.time() - start)

# correct for image scale
boxes /= scale

# visualize detections
for box, score, label in zip(boxes[0], scores[0], labels[0]):
    # scores are sorted so we can break
    if score < 0.05:
        break
        
    color = label_color(label)
    
    b = box.astype(int)
    draw_box(draw, b, color=color)
    
    caption = "{} {:.2f}%".format(labels_to_names[label], score*100)
    draw_caption(draw, b, caption)
    
plt.figure(figsize=(15, 15))
plt.axis('on')
plt.imshow(draw)
plt.show()
processing time:  0.05983281135559082
In [27]:
# load image
image = read_image_bgr('./eval_images/2020-04-16_15-58-30_CU.jpg')

# copy to draw on
draw = image.copy()
draw = cv2.cvtColor(draw, cv2.COLOR_BGR2RGB)

# preprocess image for network
image = preprocess_image(image)
image, scale = resize_image(image)

# process image
start = time.time()
boxes, scores, labels = model_t.predict_on_batch(np.expand_dims(image, axis=0))
print("processing time: ", time.time() - start)

# correct for image scale
boxes /= scale

# visualize detections
for box, score, label in zip(boxes[0], scores[0], labels[0]):
    # scores are sorted so we can break
    if score < 0.5:
        break
        
    color = label_color(label)
    
    b = box.astype(int)
    draw_box(draw, b, color=color)
    
    caption = "{} {:.2f}%".format(labels_to_names[label], score*100)
    draw_caption(draw, b, caption)
    
plt.figure(figsize=(15, 15))
plt.axis('on')
plt.imshow(draw)
plt.show()
processing time:  0.058315277099609375
In [28]:
# load image
image = read_image_bgr('./eval_images/2020-06-22_14-10-16_CU.jpg')
#image = read_image_bgr('./images/2020-04-18_15-48-16_CU.jpg')

# copy to draw on
draw = image.copy()
draw = cv2.cvtColor(draw, cv2.COLOR_BGR2RGB)

# preprocess image for network
image = preprocess_image(image)
image, scale = resize_image(image)

# process image
start = time.time()
boxes, scores, labels = model_t.predict_on_batch(np.expand_dims(image, axis=0))
print("processing time: ", time.time() - start)

# correct for image scale
boxes /= scale

# visualize detections
for box, score, label in zip(boxes[0], scores[0], labels[0]):
    # scores are sorted so we can break
    if score < 0.1: #default 0.5
        break
        
    color = label_color(label)
    
    b = box.astype(int)
    draw_box(draw, b, color=color)
    
    caption = "{} {:.2f}%".format(labels_to_names[label], score*100)
    draw_caption(draw, b, caption)
    
plt.figure(figsize=(15, 15))
plt.axis('on')
plt.imshow(draw)
plt.show()
processing time:  0.056870460510253906
In [ ]: